PrivateBin / PrivateBin

lib/Data/S3Storage.php

<?php
/**
 * S3.php
 *
 * an S3 compatible data backend for PrivateBin with CEPH/RadosGW in mind
 * see https://docs.ceph.com/en/latest/radosgw/s3/php/
 * based on lib/Data/GoogleCloudStorage.php from PrivateBin version 1.7.1
 *
 * @link      https://github.com/PrivateBin/PrivateBin
 * @copyright 2022 Felix J. Ogris (https://ogris.de/)
 * @license   https://www.opensource.org/licenses/zlib-license.php The zlib/libpng License
 * @version   1.4.1
 *
 * Installation:
 *   1. Make sure you have composer.lock and composer.json in the document root of your PasteBin
 *   2. If not, grab a copy from https://github.com/PrivateBin/PrivateBin
 *   3. As non-root user, install the AWS SDK for PHP:
 *      composer require aws/aws-sdk-php
 *      (On FreeBSD, install devel/php-composer2 prior, e.g.: make -C /usr/ports/devel/php-composer2 install clean)
 *   4. In cfg/conf.php, comment out all [model] and [model_options] settings
 *   5. Still in cfg/conf.php, add a new [model] section:
 *      [model]
 *      class = S3Storage
 *   6. Add a new [model_options] as well, e.g. for a Rados gateway as part of your CEPH cluster:
 *      [model_options]
 *      region = ""
 *      version = "2006-03-01"
 *      endpoint = "https://s3.my-ceph.invalid"
 *      use_path_style_endpoint = true
 *      bucket = "my-bucket"
 *      prefix = "privatebin"  (place all PrivateBin data beneath this prefix)
 *      accesskey = "my-rados-user"
 *      secretkey = "my-rados-pass"
 */

namespace PrivateBin\Data;

use Aws\S3\Exception\S3Exception;

The type Aws\S3\Exception\S3Exception was not found. Maybe you did not declare it correctly or list all dependencies?

use Aws\S3\S3Client;

The type Aws\S3\S3Client was not found. Maybe you did not declare it correctly or list all dependencies?

use PrivateBin\Json;

class S3Storage extends AbstractData
{
    /**
     * S3 client
     *
     * @access private
     * @var    S3Client
     */
    private $_client = null;

    /**
     * S3 client options
     *
     * @access private
     * @var    array
     */
    private $_options = array();

    /**
     * S3 bucket
     *
     * @access private
     * @var    string
     */
    private $_bucket = null;

    /**
     * S3 prefix for all PrivateBin data in this bucket
     *
     * @access private
     * @var    string
     */
    private $_prefix = '';

    /**
     * instantiates a new S3 data backend.
     *
     * @access public
     * @param array $options
     */
    public function __construct(array $options)
    {
        if (is_array($options)) {

The condition is_array($options) is always true.

            // AWS SDK will try to load credentials from environment if credentials are not passed via configuration
            // ref: https://docs.aws.amazon.com/sdk-for-php/v3/developer-guide/guide_credentials.html#default-credential-chain
            if (isset($options['accesskey']) && isset($options['secretkey'])) {
                $this->_options['credentials'] = array();

                $this->_options['credentials']['key']    = $options['accesskey'];
                $this->_options['credentials']['secret'] = $options['secretkey'];
            }
            if (array_key_exists('region', $options)) {
                $this->_options['region'] = $options['region'];
            }
            if (array_key_exists('version', $options)) {
                $this->_options['version'] = $options['version'];
            }
            if (array_key_exists('endpoint', $options)) {
                $this->_options['endpoint'] = $options['endpoint'];
            }
            if (array_key_exists('use_path_style_endpoint', $options)) {
                $this->_options['use_path_style_endpoint'] = filter_var($options['use_path_style_endpoint'], FILTER_VALIDATE_BOOLEAN);
            }
            if (array_key_exists('bucket', $options)) {
                $this->_bucket = $options['bucket'];
            }
            if (array_key_exists('prefix', $options)) {
                $this->_prefix = $options['prefix'];
            }
        }

        $this->_client = new S3Client($this->_options);
    }

    /**
     * returns all objects in the given prefix.
     *
     * @access private
     * @param $prefix string with prefix
     * @return array all objects in the given prefix
     */
    private function _listAllObjects($prefix)
    {
        $allObjects = array();
        $options    = array(
            'Bucket' => $this->_bucket,
            'Prefix' => $prefix,
        );

        do {
            $objectsListResponse = $this->_client->listObjects($options);
            $objects             = $objectsListResponse['Contents'] ?? array();
            foreach ($objects as $object) {
                $allObjects[]      = $object;
                $options['Marker'] = $object['Key'];
            }
        } while ($objectsListResponse['IsTruncated']);

        return $allObjects;
    }

    /**
     * returns the S3 storage object key for $pasteid in $this->_bucket.
     *
     * @access private
     * @param $pasteid string to get the key for
     * @return string
     */
    private function _getKey($pasteid)
    {
        if ($this->_prefix != '') {
            return $this->_prefix . '/' . $pasteid;
        }
        return $pasteid;
    }

    /**
     * Uploads the payload in the $this->_bucket under the specified key.
     * The entire payload is stored as a JSON document. The metadata is replicated
     * as the S3 object's metadata except for the fields attachment, attachmentname
     * and salt.
     *
     * @param $key string to store the payload under
     * @param $payload array to store
     * @return bool true if successful, otherwise false.
     */
    private function _upload($key, $payload)
    {
        $metadata = array_key_exists('meta', $payload) ? $payload['meta'] : array();
        unset($metadata['attachment'], $metadata['attachmentname'], $metadata['salt']);
        foreach ($metadata as $k => $v) {
            $metadata[$k] = strval($v);
        }
        try {
            $this->_client->putObject(array(
                'Bucket'      => $this->_bucket,
                'Key'         => $key,
                'Body'        => Json::encode($payload),
                'ContentType' => 'application/json',
                'Metadata'    => $metadata,
            ));
        } catch (S3Exception $e) {
            error_log('failed to upload ' . $key . ' to ' . $this->_bucket . ', ' .
                trim(preg_replace('/\s\s+/', ' ', $e->getMessage())));
            return false;
        }
        return true;
    }

    /**
     * @inheritDoc
     */
    public function create($pasteid, array $paste)
    {
        if ($this->exists($pasteid)) {
            return false;
        }

        return $this->_upload($this->_getKey($pasteid), $paste);
    }

    /**
     * @inheritDoc
     */
    public function read($pasteid)
    {
        try {
            $object = $this->_client->getObject(array(
                'Bucket' => $this->_bucket,
                'Key'    => $this->_getKey($pasteid),
            ));
            $data = $object['Body']->getContents();
            return Json::decode($data);
        } catch (S3Exception $e) {
            error_log('failed to read ' . $pasteid . ' from ' . $this->_bucket . ', ' .
                trim(preg_replace('/\s\s+/', ' ', $e->getMessage())));
            return false;
        }
    }

    /**
     * @inheritDoc
     */
    public function delete($pasteid)
    {
        $name = $this->_getKey($pasteid);

        try {
            $comments = $this->_listAllObjects($name . '/discussion/');
            foreach ($comments as $comment) {
                try {
                    $this->_client->deleteObject(array(
                        'Bucket' => $this->_bucket,
                        'Key'    => $comment['Key'],
                    ));
                } catch (S3Exception $e) {
                    // ignore if already deleted.
                }
            }
        } catch (S3Exception $e) {
            // there are no discussions associated with the paste
        }

        try {
            $this->_client->deleteObject(array(
                'Bucket' => $this->_bucket,
                'Key'    => $name,
            ));
        } catch (S3Exception $e) {
            // ignore if already deleted
        }
    }

    /**
     * @inheritDoc
     */
    public function exists($pasteid)
    {
        return $this->_client->doesObjectExistV2($this->_bucket, $this->_getKey($pasteid));
    }

    /**
     * @inheritDoc
     */
    public function createComment($pasteid, $parentid, $commentid, array $comment)
    {
        if ($this->existsComment($pasteid, $parentid, $commentid)) {
            return false;
        }
        $key = $this->_getKey($pasteid) . '/discussion/' . $parentid . '/' . $commentid;
        return $this->_upload($key, $comment);
    }

    /**
     * @inheritDoc
     */
    public function readComments($pasteid)
    {
        $comments = array();
        $prefix   = $this->_getKey($pasteid) . '/discussion/';
        try {
            $entries = $this->_listAllObjects($prefix);
            foreach ($entries as $entry) {
                $object = $this->_client->getObject(array(
                    'Bucket' => $this->_bucket,
                    'Key'    => $entry['Key'],
                ));
                $body             = JSON::decode($object['Body']->getContents());
                $items            = explode('/', $entry['Key']);
                $body['id']       = $items[3];
                $body['parentid'] = $items[2];
                $slot             = $this->getOpenSlot($comments, (int) $object['Metadata']['created']);
                $comments[$slot]  = $body;
            }
        } catch (S3Exception $e) {
            // no comments found
        }
        return $comments;
    }

    /**
     * @inheritDoc
     */
    public function existsComment($pasteid, $parentid, $commentid)
    {
        $name = $this->_getKey($pasteid) . '/discussion/' . $parentid . '/' . $commentid;
        return $this->_client->doesObjectExistV2($this->_bucket, $name);
    }

    /**
     * @inheritDoc
     */
    public function purgeValues($namespace, $time)
    {
        $path = $this->_prefix;
        if ($path != '') {
            $path .= '/';
        }
        $path .= 'config/' . $namespace;

        try {
            foreach ($this->_listAllObjects($path) as $object) {
                $name = $object['Key'];
                if (strlen($name) > strlen($path) && substr($name, strlen($path), 1) !== '/') {
                    continue;
                }
                $head = $this->_client->headObject(array(
                    'Bucket' => $this->_bucket,
                    'Key'    => $name,
                ));
                if ($head->get('Metadata') != null && array_key_exists('value', $head->get('Metadata'))) {
                    $value = $head->get('Metadata')['value'];
                    if (is_numeric($value) && intval($value) < $time) {
                        try {
                            $this->_client->deleteObject(array(
                                'Bucket' => $this->_bucket,
                                'Key'    => $name,
                            ));
                        } catch (S3Exception $e) {
                            // deleted by another instance.
                        }
                    }
                }
            }
        } catch (S3Exception $e) {
            // no objects in the bucket yet
        }
    }

    /**
     * For S3, the value will also be stored in the metadata for the
     * namespaces traffic_limiter and purge_limiter.
     * @inheritDoc
     */
    public function setValue($value, $namespace, $key = '')
    {
        $prefix = $this->_prefix;
        if ($prefix != '') {
            $prefix .= '/';
        }

        if ($key === '') {
            $key = $prefix . 'config/' . $namespace;
        } else {
            $key = $prefix . 'config/' . $namespace . '/' . $key;
        }

        $metadata = array('namespace' => $namespace);
        if ($namespace != 'salt') {
            $metadata['value'] = strval($value);
        }
        try {
            $this->_client->putObject(array(
                'Bucket'      => $this->_bucket,
                'Key'         => $key,
                'Body'        => $value,
                'ContentType' => 'application/json',
                'Metadata'    => $metadata,
            ));
        } catch (S3Exception $e) {
            error_log('failed to set key ' . $key . ' to ' . $this->_bucket . ', ' .
                trim(preg_replace('/\s\s+/', ' ', $e->getMessage())));
            return false;
        }
        return true;
    }

    /**
     * @inheritDoc
     */
    public function getValue($namespace, $key = '')
    {
        $prefix = $this->_prefix;
        if ($prefix != '') {
            $prefix .= '/';
        }

        if ($key === '') {
            $key = $prefix . 'config/' . $namespace;
        } else {
            $key = $prefix . 'config/' . $namespace . '/' . $key;
        }

        try {
            $object = $this->_client->getObject(array(
                'Bucket' => $this->_bucket,
                'Key'    => $key,
            ));
            return $object['Body']->getContents();
        } catch (S3Exception $e) {
            return '';
        }
    }

    /**
     * @inheritDoc
     */
    protected function _getExpiredPastes($batchsize)
    {
        $expired = array();
        $now     = time();
        $prefix  = $this->_prefix;
        if ($prefix != '') {
            $prefix .= '/';
        }

        try {
            foreach ($this->_listAllObjects($prefix) as $object) {
                $head = $this->_client->headObject(array(
                    'Bucket' => $this->_bucket,
                    'Key'    => $object['Key'],
                ));
                if ($head->get('Metadata') != null && array_key_exists('expire_date', $head->get('Metadata'))) {
                    $expire_at = intval($head->get('Metadata')['expire_date']);
                    if ($expire_at != 0 && $expire_at < $now) {
                        array_push($expired, $object['Key']);
                    }
                }

                if (count($expired) > $batchsize) {
                    break;
                }
            }
        } catch (S3Exception $e) {
            // no objects in the bucket yet
        }
        return $expired;
    }

    /**
     * @inheritDoc
     */
    public function getAllPastes()
    {
        $pastes = array();
        $prefix = $this->_prefix;
        if ($prefix != '') {
            $prefix .= '/';
        }

        try {
            foreach ($this->_listAllObjects($prefix) as $object) {
                $candidate = substr($object['Key'], strlen($prefix));
                if (strpos($candidate, '/') === false) {
                    $pastes[] = $candidate;
                }
            }
        } catch (S3Exception $e) {
            // no objects in the bucket yet
        }
        return $pastes;
    }
}