DataloadService::copy() - Code Metrics - Inspection of "Merge remote-tracking branch 'origin/master'" - Rello/analytics - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Push — master ( 952c0a...1807d7 )

by Marcel

created 2024-01-18 10:23 UTC

DataloadService::copy() A

↳ Parent: DataloadService

Complexity

Conditions	1
Paths	1

Size

Total Lines	3
Code Lines	1

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
cc	1
eloc	1
nc	1
nop	1
dl	0
loc	3
rs	10
c	0
b	0
f	0

<?php
/**
 * Analytics
 *
 * This file is licensed under the Affero General Public License version 3 or
 * later. See the LICENSE.md file.
 *
 * @author Marcel Scherello <[email protected]>
 * @copyright 2019-2022 Marcel Scherello
 */

namespace OCA\Analytics\Service;

use Exception;
use OCA\Analytics\Activity\ActivityManager;
use OCA\Analytics\Controller\DatasourceController;
use OCA\Analytics\Notification\NotificationManager;
use OCA\Analytics\Db\DataloadMapper;
use OCP\AppFramework\Http\NotFoundResponse;
filter:
    dependency_paths: ["lib/*"]
use OCP\Files\NotFoundException;
filter:
    dependency_paths: ["lib/*"]
use OCP\IL10N;
filter:
    dependency_paths: ["lib/*"]
use Psr\Log\LoggerInterface;
filter:
    dependency_paths: ["lib/*"]

class DataloadService
{
    private $userId;
    private $logger;
    private $StorageService;
    private $DatasourceController;
    private $ActivityManager;
    private $ReportService;
    private $DatasetService;
    private $VariableService;
    private $l10n;
    private $DataloadMapper;
    private $NotificationManager;

    public function __construct(
        $userId,
        IL10N $l10n,
        LoggerInterface $logger,
        ActivityManager $ActivityManager,
        DatasourceController $DatasourceController,
        ReportService $ReportService,
        DatasetService $DatasetService,
        StorageService $StorageService,
        VariableService $VariableService,
        NotificationManager $NotificationManager,
        DataloadMapper $DataloadMapper
    )
    {
        $this->userId = $userId;
        $this->l10n = $l10n;
        $this->logger = $logger;
        $this->StorageService = $StorageService;
        $this->ActivityManager = $ActivityManager;
        $this->DatasourceController = $DatasourceController;
        $this->ReportService = $ReportService;
        $this->DatasetService = $DatasetService;
        $this->VariableService = $VariableService;
        $this->NotificationManager = $NotificationManager;
        $this->DataloadMapper = $DataloadMapper;
    }

    // Data loads
    // Data loads
    // Data loads

    /**
     * create a new data load
     *
     * @param $datasetId
     * @param $reportId
     * @param int $datasourceId
     * @return int
     * @throws \OCP\DB\Exception
     */
    public function create($datasetId, int $datasourceId)
    {
        return $this->DataloadMapper->create((int)$datasetId, $datasourceId);
    }

    /**
     * get all data loads for a dataset or report
     *
     * @param int $datasetId
     * @param $reportId
     * @return array
     */
    public function read($datasetId)
    {
        return $this->DataloadMapper->read((int)$datasetId);
    }

    /**
     * update data load
     *
     * @param int $dataloadId
     * @param $name
     * @param $option
     * @param $schedule
     * @return bool
     */
    public function update(int $dataloadId, $name, $option, $schedule)
    {
        $array = json_decode($option, true);
        foreach ($array as $key => $value) {
            $array[$key] = htmlspecialchars($value, ENT_NOQUOTES, 'UTF-8');
        }
        $option = json_encode($array);

        return $this->DataloadMapper->update($dataloadId, $name, $option, $schedule);
    }

    /**
     * copy a data load
     *
     * @param int $dataloadId
     * @return bool
     */
    public function copy(int $dataloadId)
    {
        return $this->DataloadMapper->copy($dataloadId);
    }

    /**
     * delete a data load
     *
     * @param int $dataloadId
     * @return bool
     */
    public function delete(int $dataloadId)
    {
        return $this->DataloadMapper->delete($dataloadId);
    }

    /**
     * execute all data loads depending on their schedule
     * Daily or Hourly
     *
     * @param $schedule
     * @return void
     * @throws Exception
     */
    public function executeBySchedule($schedule)
    {
        $schedules = $this->DataloadMapper->getDataloadBySchedule($schedule);
        foreach ($schedules as $dataload) {
            $result = $this->execute($dataload['id']);
            if ($result['error'] !== 0) {
                // if the data source produced an error, a notification needs to be triggered
                $dataset = $this->DatasetService->read($dataload['dataset']);
                $this->NotificationManager->triggerNotification(NotificationManager::DATALOAD_ERROR, $dataload['dataset'], $dataload['id'], ['dataloadName' => $dataload['name'], 'datasetName' => $dataset['name']], $dataload['user_id']);
            }
        }
    }

    /**
     * execute a data load from data source and store into dataset
     *
     * @param int $dataloadId
     * @return array
     * @throws Exception
     */
    public function execute(int $dataloadId)
    {
        $bulkSize = 500;
        $insert = $update = $error = $delete = $currentCount = 0;
        $bulkInsert = null;
        $aggregation = null;

        // get the data from the datasource
        $result = $this->getDataFromDatasource($dataloadId);

        // dont continue in case of datasource error
        if ($result['error'] !== 0) {
            return [
                'insert' => $insert,
                'update' => $update,
                'delete' => $delete,
                'error' => 1,
            ];
        }

        // get the meta data
        $dataloadMetadata = $this->DataloadMapper->getDataloadById($dataloadId);
        $option = json_decode($dataloadMetadata['option'], true);
        $datasetId = $dataloadMetadata['dataset'];

        // this is a deletion request. Just run the deletion and stop after that with a return.
        if ($dataloadMetadata['datasource'] === 0) {
            // deletion jobs are using the same dimension/option/value settings a report filter
            $filter = array();
            $filter['filteroptions'] = '{"filter":{"' . $option['filterDimension'] . '":{"option":"' . $option['filterOption'] . '","value":"' . $option['filterValue'] . '"}}}';
            // Text variables like %xx% could be in use here
            $filter = $this->VariableService->replaceFilterVariables($filter);

            $records = $this->StorageService->deleteWithFilter($dataloadMetadata['dataset'], json_decode($filter['filteroptions'], true));

            return [
                'insert' => $insert,
                'update' => $update,
                'delete' => $records,
                'error' => $error,
            ];
        }

        // "delete all date before loading" is true in the data source options
        // in this case, bulkInsert is additionally set to true. Then no further checks for existing records are needed
        // to reduce db selects
        if (isset($option['delete']) and $option['delete'] === 'true') {
            $this->StorageService->delete($datasetId, '*', '*', $dataloadMetadata['user_id']);
            $bulkInsert = true;
        }

        // if the data set has no data, it is the same as the delete all option
        // in this case, bulkInsert is set to true. Then no further checks for existing records are needed
        // to reduce db selects
        $numberOfRecords = $this->StorageService->getRecordCount($datasetId, $dataloadMetadata['user_id']);
        if ($numberOfRecords['count'] === 0) {
            $bulkInsert = true;
        }

        // Feature not yet available
        if (isset($option['aggregation']) and $option['aggregation'] !== 'overwrite') {
            $aggregation = $option['aggregation'];
        }

        // collect mass updates to reduce statements to the database
        $this->DataloadMapper->beginTransaction();
        foreach ($result['data'] as $row) {
            // only one column is not possible
            if (count($row) === 1) {
                $this->logger->info('loading data with only one column is not possible. This is a data load for the dataset: ' . $datasetId);
                $error = $error + 1;
                continue;
            }
            // if data source only delivers 2 columns, the value needs to be in the last one
            if (count($row) === 2) {
                $row[2] = $row[1];
                $row[1] = null;
            }

            $action = $this->StorageService->update($datasetId, $row[0], $row[1], $row[2], $dataloadMetadata['user_id'], $bulkInsert, $aggregation);
            $insert = $insert + $action['insert'];
            $update = $update + $action['update'];
            $error = $error + $action['error'];

            if ($currentCount % $bulkSize === 0) {
                $this->DataloadMapper->commit();
                $this->DataloadMapper->beginTransaction();
            }
            if ($action['error'] === 0) $currentCount++;
        }
        $this->DataloadMapper->commit();

        $result = [
            'insert' => $insert,
            'update' => $update,
            'delete' => $delete,
            'error' => $error,
        ];

        $this->ActivityManager->triggerEvent($datasetId, ActivityManager::OBJECT_DATA, ActivityManager::SUBJECT_DATA_ADD_DATALOAD, $dataloadMetadata['user_id']);
        return $result;
    }

    /**
     * get the data from datasource
     * to be used in simulation or execution
     *
     * @param int $dataloadId
     * @return array|NotFoundResponse
     * @throws NotFoundResponse
     * @throws \OCP\DB\Exception
     */
    public function getDataFromDatasource(int $dataloadId)
    {
        $dataloadMetadata = $this->DataloadMapper->getDataloadById($dataloadId);

        if (!empty($dataloadMetadata)) {

            if ($dataloadMetadata['datasource'] !== 0) {
                $dataloadMetadata['link'] = $dataloadMetadata['option']; //remap until data source table is renamed link=>option

                $result = $this->DatasourceController->read((int)$dataloadMetadata['datasource'], $dataloadMetadata);
                $result['datasetId'] = $dataloadMetadata['dataset'];
            } else {
                // this is a deletion request. Just run the simulation and return the possible row count in the expected result array
                $option = json_decode($dataloadMetadata['option'], true);

                // deletion jobs are using the same dimension/option/value settings a report filter
                $filter = array();
                $filter['filteroptions'] = '{"filter":{"' . $option['filterDimension'] . '":{"option":"' . $option['filterOption'] . '","value":"' . $option['filterValue'] . '"}}}';
                // Text variables like %xx% could be in use here
                $filter = $this->VariableService->replaceFilterVariables($filter);

                $result = [
                    'header' => '',
                    'dimensions' => '',
                    'data' => $this->StorageService->deleteWithFilterSimulate($dataloadMetadata['dataset'], json_decode($filter['filteroptions'], true)),
                    'error' => 0,
                ];
            }

            return $result;
        } else {
            return new NotFoundResponse();
        }
    }

    // Data Manipulation
    // Data Manipulation
    // Data Manipulation

    /**
     * update data from input form
     *
     * @NoAdminRequired
     * @param int $objectId
     * @param $dimension1
     * @param $dimension2
     * @param $value
     * @param bool $isDataset
     * @return array|false
     * @throws \OCP\DB\Exception
     */
    public function updateData(int $objectId, $dimension1, $dimension2, $value, bool $isDataset)
    {
        $dataset = $this->getDatasetId($objectId, $isDataset);

        if ($dataset != '') {
            $insert = $update = $errorMessage = 0;
            $action = array();
            $value = $this->floatvalue($value);
            if ($value === false) {
                $errorMessage = $this->l10n->t('3rd field must be a valid number');
            } else {
                $action = $this->StorageService->update($dataset, $dimension1, $dimension2, $value);
                $insert = $insert + $action['insert'];
                $update = $update + $action['update'];
            }

            $result = [
                'insert' => $insert,
                'update' => $update,
                'error' => $errorMessage,
                'validate' => $action['validate'],
            ];

            if ($errorMessage === 0) $this->ActivityManager->triggerEvent($dataset, ActivityManager::OBJECT_DATA, ActivityManager::SUBJECT_DATA_ADD);
            return $result;
        } else {
            return false;
        }
    }

    /**
     * Simulate delete data from input form
     *
     * @NoAdminRequired
     * @param int $objectId
     * @param $dimension1
     * @param $dimension2
     * @param bool $isDataset
     * @return array|false
     * @throws \OCP\DB\Exception
     */
    public function deleteDataSimulate(int $objectId, $dimension1, $dimension2, bool $isDataset)
    {
        $dataset = $this->getDatasetId($objectId, $isDataset);
        if ($dataset != '') {
            $result = $this->StorageService->deleteSimulate($dataset, $dimension1, $dimension2);
            return ['delete' => $result];
        } else {
            return false;
        }
    }

    /**
     * delete data from input form
     *
     * @NoAdminRequired
     * @param int $objectId
     * @param $dimension1
     * @param $dimension2
     * @param bool $isDataset
     * @return array|false
     * @throws \OCP\DB\Exception
     */
    public function deleteData(int $objectId, $dimension1, $dimension2, bool $isDataset)
    {
        $dataset = $this->getDatasetId($objectId, $isDataset);
        if ($dataset != '') {
            $result = $this->StorageService->delete($dataset, $dimension1, $dimension2);
            return ['delete' => $result];
        } else {
            return false;
        }
    }

    /**
     * Import clipboard data
     *
     * @NoAdminRequired
     * @param int $objectId
     * @param $import
     * @param bool $isDataset
     * @return array|false
     * @throws \OCP\DB\Exception
     */
    public function importClipboard($objectId, $import, bool $isDataset)
    {
        $dataset = $this->getDatasetId($objectId, $isDataset);
        if ($dataset != '') {
            $insert = $update = $errorMessage = $errorCounter = 0;
            $delimiter = '';

            if ($import === '') {
                $errorMessage = $this->l10n->t('No data');
            } else {
                $delimiter = $this->detectDelimiter($import);
                $rows = str_getcsv($import, "\n");

                foreach ($rows as &$row) {
                    $row = str_getcsv($row, $delimiter);
                    $numberOfColumns = count($row);
                    // last column needs to be a float
                    $row[2] = $this->floatvalue($row[$numberOfColumns - 1]);
                    if ($row[2] === false) {
                        $errorCounter++;
                    } else {
                        if ($numberOfColumns < 3) $row[1] = null;
                        $action = $this->StorageService->update($dataset, $row[0], $row[1], $row[2]);
                        $insert = $insert + $action['insert'];
                        $update = $update + $action['update'];
                    }
                    if ($errorCounter === 2) {
                        // first error is ignored; might be due to header row
                        $errorMessage = $this->l10n->t('Last field must be a valid number');
                        break;
                    }
                }
            }

            $result = [
                'insert' => $insert,
                'update' => $update,
                'delimiter' => $delimiter,
                'error' => $errorMessage,
            ];

            if ($errorMessage === 0) $this->ActivityManager->triggerEvent($dataset, ActivityManager::OBJECT_DATA, ActivityManager::SUBJECT_DATA_ADD_IMPORT);
            return $result;
        } else {
            return false;
        }
    }

    /**
     * Import data into dataset from an internal or external file
     *
     * @NoAdminRequired
     * @param int $objectId
     * @param $path
     * @param bool $isDataset
     * @return array|false
     * @throws \OCP\DB\Exception
     */
    public function importFile(int $objectId, $path, bool $isDataset)
    {
        $dataset = $this->getDatasetId($objectId, $isDataset);
        if ($dataset != '') {
            $insert = $update = 0;
            $reportMetadata = array();
            $reportMetadata['link'] = $path;
            $reportMetadata['user_id'] = $this->userId;
            $result = $this->DatasourceController->read(DatasourceController::DATASET_TYPE_FILE, $reportMetadata);

            if ($result['error'] === 0) {
                foreach ($result['data'] as &$row) {
                    $action = $this->StorageService->update($dataset, $row[0], $row[1], $row[2]);
                    $insert = $insert + $action['insert'];
                    $update = $update + $action['update'];
                }
            }

            $result = [
                'insert' => $insert,
                'update' => $update,
                'error' => $result['error'],
            ];

            if ($result['error'] === 0) $this->ActivityManager->triggerEvent($dataset, ActivityManager::OBJECT_DATA, ActivityManager::SUBJECT_DATA_ADD_IMPORT);
            return $result;
        } else {
            return false;
        }
    }

    private function getDatasetId($objectId, bool $isDataset)
    {
        if ($isDataset) {
            $dataset = $objectId;
        } else {
            $reportMetadata = $this->ReportService->read($objectId);
            $dataset = (int)$reportMetadata['dataset'];
        }
        return $dataset;
    }

    private function detectDelimiter($data): string
    {
        $delimiters = ["\t", ";", "|", ","];
        $data_2 = array();
        $delimiter = $delimiters[0];
        foreach ($delimiters as $d) {
            $firstRow = str_getcsv($data, "\n")[0];
            $data_1 = str_getcsv($firstRow, $d);
            if (sizeof($data_1) > sizeof($data_2)) {
                $delimiter = $d;
                $data_2 = $data_1;
            }
        }
        return $delimiter;
    }

    private function floatvalue($val)
    {
        // if value is a 3 digit comma number with one leading zero like 0,111, it should not go through the 1000 separator removal
        if (preg_match('/(?<=\b0)\,(?=\d{3}\b)/', $val) === 0 && preg_match('/(?<=\b0)\.(?=\d{3}\b)/', $val) === 0) {
            // remove , as 1000 separator
            $val = preg_replace('/(?<=\d)\,(?=\d{3}\b)/', '', $val);
            // remove . as 1000 separator
            $val = preg_replace('/(?<=\d)\.(?=\d{3}\b)/', '', $val);
        }
        // convert remaining comma to decimal point
        $val = str_replace(",", ".", $val);
        if (is_numeric($val)) {
            return number_format(floatval($val), 2, '.', '');
        } else {
            return false;
        }
    }

}

1			<?php
2			/**
3			* Analytics
4			*
5			* This file is licensed under the Affero General Public License version 3 or
6			* later. See the LICENSE.md file.
7			*
8			* @author Marcel Scherello <[email protected]>
9			* @copyright 2019-2022 Marcel Scherello
10			*/
11
12			namespace OCA\Analytics\Service;
13
14			use Exception;
15			use OCA\Analytics\Activity\ActivityManager;
16			use OCA\Analytics\Controller\DatasourceController;
17			use OCA\Analytics\Notification\NotificationManager;
18			use OCA\Analytics\Db\DataloadMapper;
19			use OCP\AppFramework\Http\NotFoundResponse;
			0 ignored issues – show Bug introduced 2023-08-25 06:43 UTC by Report Bug Copy Issue Report The type `OCP\AppFramework\Http\NotFoundResponse` was not found. Maybe you did not declare it correctly or list all dependencies? The issue could also be caused by a filter entry in the build configuration. If the path has been excluded in your configuration, e.g. `excluded_paths: ["lib/"]`, you can move it to the dependency path list as follows: filter: dependency_paths: ["lib/"] For further information see https://scrutinizer-ci.com/docs/tools/php/php-scrutinizer/#list-dependency-paths Loading history...
20			use OCP\Files\NotFoundException;
			0 ignored issues – show Bug introduced 2023-08-25 06:43 UTC by Report Bug Copy Issue Report The type `OCP\Files\NotFoundException` was not found. Maybe you did not declare it correctly or list all dependencies? The issue could also be caused by a filter entry in the build configuration. If the path has been excluded in your configuration, e.g. `excluded_paths: ["lib/"]`, you can move it to the dependency path list as follows: filter: dependency_paths: ["lib/"] For further information see https://scrutinizer-ci.com/docs/tools/php/php-scrutinizer/#list-dependency-paths Loading history...
21			use OCP\IL10N;
			0 ignored issues – show Bug introduced 2023-08-25 06:43 UTC by Report Bug Copy Issue Report The type `OCP\IL10N` was not found. Maybe you did not declare it correctly or list all dependencies? The issue could also be caused by a filter entry in the build configuration. If the path has been excluded in your configuration, e.g. `excluded_paths: ["lib/"]`, you can move it to the dependency path list as follows: filter: dependency_paths: ["lib/"] For further information see https://scrutinizer-ci.com/docs/tools/php/php-scrutinizer/#list-dependency-paths Loading history...
22			use Psr\Log\LoggerInterface;
			0 ignored issues – show Bug introduced 2023-08-25 06:43 UTC by Report Bug Copy Issue Report The type `Psr\Log\LoggerInterface` was not found. Maybe you did not declare it correctly or list all dependencies? The issue could also be caused by a filter entry in the build configuration. If the path has been excluded in your configuration, e.g. `excluded_paths: ["lib/"]`, you can move it to the dependency path list as follows: filter: dependency_paths: ["lib/"] For further information see https://scrutinizer-ci.com/docs/tools/php/php-scrutinizer/#list-dependency-paths Loading history...
23
24			class DataloadService
25			{
26			private $userId;
27			private $logger;
28			private $StorageService;
29			private $DatasourceController;
30			private $ActivityManager;
31			private $ReportService;
32			private $DatasetService;
33			private $VariableService;
34			private $l10n;
35			private $DataloadMapper;
36			private $NotificationManager;
37
38			public function __construct(
39			$userId,
40			IL10N $l10n,
41			LoggerInterface $logger,
42			ActivityManager $ActivityManager,
43			DatasourceController $DatasourceController,
44			ReportService $ReportService,
45			DatasetService $DatasetService,
46			StorageService $StorageService,
47			VariableService $VariableService,
48			NotificationManager $NotificationManager,
49			DataloadMapper $DataloadMapper
50			)
51			{
52			$this->userId = $userId;
53			$this->l10n = $l10n;
54			$this->logger = $logger;
55			$this->StorageService = $StorageService;
56			$this->ActivityManager = $ActivityManager;
57			$this->DatasourceController = $DatasourceController;
58			$this->ReportService = $ReportService;
59			$this->DatasetService = $DatasetService;
60			$this->VariableService = $VariableService;
61			$this->NotificationManager = $NotificationManager;
62			$this->DataloadMapper = $DataloadMapper;
63			}
64
65			// Data loads
66			// Data loads
67			// Data loads
68
69			/**
70			* create a new data load
71			*
72			* @param $datasetId
73			* @param $reportId
74			* @param int $datasourceId
75			* @return int
76			* @throws \OCP\DB\Exception
77			*/
78			public function create($datasetId, int $datasourceId)
79			{
80			return $this->DataloadMapper->create((int)$datasetId, $datasourceId);
81			}
82
83			/**
84			* get all data loads for a dataset or report
85			*
86			* @param int $datasetId
87			* @param $reportId
88			* @return array
89			*/
90			public function read($datasetId)
91			{
92			return $this->DataloadMapper->read((int)$datasetId);
93			}
94
95			/**
96			* update data load
97			*
98			* @param int $dataloadId
99			* @param $name
100			* @param $option
101			* @param $schedule
102			* @return bool
103			*/
104			public function update(int $dataloadId, $name, $option, $schedule)
105			{
106			$array = json_decode($option, true);
107			foreach ($array as $key => $value) {
108			$array[$key] = htmlspecialchars($value, ENT_NOQUOTES, 'UTF-8');
109			}
110			$option = json_encode($array);
111
112			return $this->DataloadMapper->update($dataloadId, $name, $option, $schedule);
113			}
114
115			/**
116			* copy a data load
117			*
118			* @param int $dataloadId
119			* @return bool
120			*/
121			public function copy(int $dataloadId)
122			{
123			return $this->DataloadMapper->copy($dataloadId);
124			}
125
126			/**
127			* delete a data load
128			*
129			* @param int $dataloadId
130			* @return bool
131			*/
132			public function delete(int $dataloadId)
133			{
134			return $this->DataloadMapper->delete($dataloadId);
135			}
136
137			/**
138			* execute all data loads depending on their schedule
139			* Daily or Hourly
140			*
141			* @param $schedule
142			* @return void
143			* @throws Exception
144			*/
145			public function executeBySchedule($schedule)
146			{
147			$schedules = $this->DataloadMapper->getDataloadBySchedule($schedule);
148			foreach ($schedules as $dataload) {
149			$result = $this->execute($dataload['id']);
150			if ($result['error'] !== 0) {
151			// if the data source produced an error, a notification needs to be triggered
152			$dataset = $this->DatasetService->read($dataload['dataset']);
153			$this->NotificationManager->triggerNotification(NotificationManager::DATALOAD_ERROR, $dataload['dataset'], $dataload['id'], ['dataloadName' => $dataload['name'], 'datasetName' => $dataset['name']], $dataload['user_id']);
154			}
155			}
156			}
157
158			/**
159			* execute a data load from data source and store into dataset
160			*
161			* @param int $dataloadId
162			* @return array
163			* @throws Exception
164			*/
165			public function execute(int $dataloadId)
166			{
167			$bulkSize = 500;
168			$insert = $update = $error = $delete = $currentCount = 0;
169			$bulkInsert = null;
170			$aggregation = null;
171
172			// get the data from the datasource
173			$result = $this->getDataFromDatasource($dataloadId);
174
175			// dont continue in case of datasource error
176			if ($result['error'] !== 0) {
177			return [
178			'insert' => $insert,
179			'update' => $update,
180			'delete' => $delete,
181			'error' => 1,
182			];
183			}
184
185			// get the meta data
186			$dataloadMetadata = $this->DataloadMapper->getDataloadById($dataloadId);
187			$option = json_decode($dataloadMetadata['option'], true);
188			$datasetId = $dataloadMetadata['dataset'];
189
190			// this is a deletion request. Just run the deletion and stop after that with a return.
191			if ($dataloadMetadata['datasource'] === 0) {
192			// deletion jobs are using the same dimension/option/value settings a report filter
193			$filter = array();
194			$filter['filteroptions'] = '{"filter":{"' . $option['filterDimension'] . '":{"option":"' . $option['filterOption'] . '","value":"' . $option['filterValue'] . '"}}}';
195			// Text variables like %xx% could be in use here
196			$filter = $this->VariableService->replaceFilterVariables($filter);
197
198			$records = $this->StorageService->deleteWithFilter($dataloadMetadata['dataset'], json_decode($filter['filteroptions'], true));
199
200			return [
201			'insert' => $insert,
202			'update' => $update,
203			'delete' => $records,
204			'error' => $error,
205			];
206			}
207
208			// "delete all date before loading" is true in the data source options
209			// in this case, bulkInsert is additionally set to true. Then no further checks for existing records are needed
210			// to reduce db selects
211			if (isset($option['delete']) and $option['delete'] === 'true') {
212			$this->StorageService->delete($datasetId, '', '', $dataloadMetadata['user_id']);
213			$bulkInsert = true;
214			}
215
216			// if the data set has no data, it is the same as the delete all option
217			// in this case, bulkInsert is set to true. Then no further checks for existing records are needed
218			// to reduce db selects
219			$numberOfRecords = $this->StorageService->getRecordCount($datasetId, $dataloadMetadata['user_id']);
220			if ($numberOfRecords['count'] === 0) {
221			$bulkInsert = true;
222			}
223
224			// Feature not yet available
225			if (isset($option['aggregation']) and $option['aggregation'] !== 'overwrite') {
226			$aggregation = $option['aggregation'];
227			}
228
229			// collect mass updates to reduce statements to the database
230			$this->DataloadMapper->beginTransaction();
231			foreach ($result['data'] as $row) {
232			// only one column is not possible
233			if (count($row) === 1) {
234			$this->logger->info('loading data with only one column is not possible. This is a data load for the dataset: ' . $datasetId);
235			$error = $error + 1;
236			continue;
237			}
238			// if data source only delivers 2 columns, the value needs to be in the last one
239			if (count($row) === 2) {
240			$row[2] = $row[1];
241			$row[1] = null;
242			}
243
244			$action = $this->StorageService->update($datasetId, $row[0], $row[1], $row[2], $dataloadMetadata['user_id'], $bulkInsert, $aggregation);
245			$insert = $insert + $action['insert'];
246			$update = $update + $action['update'];
247			$error = $error + $action['error'];
248
249			if ($currentCount % $bulkSize === 0) {
250			$this->DataloadMapper->commit();
251			$this->DataloadMapper->beginTransaction();
252			}
253			if ($action['error'] === 0) $currentCount++;
254			}
255			$this->DataloadMapper->commit();
256
257			$result = [
258			'insert' => $insert,
259			'update' => $update,
260			'delete' => $delete,
261			'error' => $error,
262			];
263
264			$this->ActivityManager->triggerEvent($datasetId, ActivityManager::OBJECT_DATA, ActivityManager::SUBJECT_DATA_ADD_DATALOAD, $dataloadMetadata['user_id']);
265			return $result;
266			}
267
268			/**
269			* get the data from datasource
270			* to be used in simulation or execution
271			*
272			* @param int $dataloadId
273			* @return array\|NotFoundResponse
274			* @throws NotFoundResponse
275			* @throws \OCP\DB\Exception
276			*/
277			public function getDataFromDatasource(int $dataloadId)
278			{
279			$dataloadMetadata = $this->DataloadMapper->getDataloadById($dataloadId);
280
281			if (!empty($dataloadMetadata)) {
282
283			if ($dataloadMetadata['datasource'] !== 0) {
284			$dataloadMetadata['link'] = $dataloadMetadata['option']; //remap until data source table is renamed link=>option
285
286			$result = $this->DatasourceController->read((int)$dataloadMetadata['datasource'], $dataloadMetadata);
287			$result['datasetId'] = $dataloadMetadata['dataset'];
288			} else {
289			// this is a deletion request. Just run the simulation and return the possible row count in the expected result array
290			$option = json_decode($dataloadMetadata['option'], true);
291
292			// deletion jobs are using the same dimension/option/value settings a report filter
293			$filter = array();
294			$filter['filteroptions'] = '{"filter":{"' . $option['filterDimension'] . '":{"option":"' . $option['filterOption'] . '","value":"' . $option['filterValue'] . '"}}}';
295			// Text variables like %xx% could be in use here
296			$filter = $this->VariableService->replaceFilterVariables($filter);
297
298			$result = [
299			'header' => '',
300			'dimensions' => '',
301			'data' => $this->StorageService->deleteWithFilterSimulate($dataloadMetadata['dataset'], json_decode($filter['filteroptions'], true)),
302			'error' => 0,
303			];
304			}
305
306			return $result;
307			} else {
308			return new NotFoundResponse();
309			}
310			}
311
312			// Data Manipulation
313			// Data Manipulation
314			// Data Manipulation
315
316			/**
317			* update data from input form
318			*
319			* @NoAdminRequired
320			* @param int $objectId
321			* @param $dimension1
322			* @param $dimension2
323			* @param $value
324			* @param bool $isDataset
325			* @return array\|false
326			* @throws \OCP\DB\Exception
327			*/
328			public function updateData(int $objectId, $dimension1, $dimension2, $value, bool $isDataset)
329			{
330			$dataset = $this->getDatasetId($objectId, $isDataset);
331
332			if ($dataset != '') {
333			$insert = $update = $errorMessage = 0;
334			$action = array();
335			$value = $this->floatvalue($value);
336			if ($value === false) {
337			$errorMessage = $this->l10n->t('3rd field must be a valid number');
338			} else {
339			$action = $this->StorageService->update($dataset, $dimension1, $dimension2, $value);
340			$insert = $insert + $action['insert'];
341			$update = $update + $action['update'];
342			}
343
344			$result = [
345			'insert' => $insert,
346			'update' => $update,
347			'error' => $errorMessage,
348			'validate' => $action['validate'],
349			];
350
351			if ($errorMessage === 0) $this->ActivityManager->triggerEvent($dataset, ActivityManager::OBJECT_DATA, ActivityManager::SUBJECT_DATA_ADD);
352			return $result;
353			} else {
354			return false;
355			}
356			}
357
358			/**
359			* Simulate delete data from input form
360			*
361			* @NoAdminRequired
362			* @param int $objectId
363			* @param $dimension1
364			* @param $dimension2
365			* @param bool $isDataset
366			* @return array\|false
367			* @throws \OCP\DB\Exception
368			*/
369			public function deleteDataSimulate(int $objectId, $dimension1, $dimension2, bool $isDataset)
370			{
371			$dataset = $this->getDatasetId($objectId, $isDataset);
372			if ($dataset != '') {
373			$result = $this->StorageService->deleteSimulate($dataset, $dimension1, $dimension2);
374			return ['delete' => $result];
375			} else {
376			return false;
377			}
378			}
379
380			/**
381			* delete data from input form
382			*
383			* @NoAdminRequired
384			* @param int $objectId
385			* @param $dimension1
386			* @param $dimension2
387			* @param bool $isDataset
388			* @return array\|false
389			* @throws \OCP\DB\Exception
390			*/
391			public function deleteData(int $objectId, $dimension1, $dimension2, bool $isDataset)
392			{
393			$dataset = $this->getDatasetId($objectId, $isDataset);
394			if ($dataset != '') {
395			$result = $this->StorageService->delete($dataset, $dimension1, $dimension2);
396			return ['delete' => $result];
397			} else {
398			return false;
399			}
400			}
401
402			/**
403			* Import clipboard data
404			*
405			* @NoAdminRequired
406			* @param int $objectId
407			* @param $import
408			* @param bool $isDataset
409			* @return array\|false
410			* @throws \OCP\DB\Exception
411			*/
412			public function importClipboard($objectId, $import, bool $isDataset)
413			{
414			$dataset = $this->getDatasetId($objectId, $isDataset);
415			if ($dataset != '') {
416			$insert = $update = $errorMessage = $errorCounter = 0;
417			$delimiter = '';
418
419			if ($import === '') {
420			$errorMessage = $this->l10n->t('No data');
421			} else {
422			$delimiter = $this->detectDelimiter($import);
423			$rows = str_getcsv($import, "\n");
424
425			foreach ($rows as &$row) {
426			$row = str_getcsv($row, $delimiter);
427			$numberOfColumns = count($row);
428			// last column needs to be a float
429			$row[2] = $this->floatvalue($row[$numberOfColumns - 1]);
430			if ($row[2] === false) {
431			$errorCounter++;
432			} else {
433			if ($numberOfColumns < 3) $row[1] = null;
434			$action = $this->StorageService->update($dataset, $row[0], $row[1], $row[2]);
435			$insert = $insert + $action['insert'];
436			$update = $update + $action['update'];
437			}
438			if ($errorCounter === 2) {
439			// first error is ignored; might be due to header row
440			$errorMessage = $this->l10n->t('Last field must be a valid number');
441			break;
442			}
443			}
444			}
445
446			$result = [
447			'insert' => $insert,
448			'update' => $update,
449			'delimiter' => $delimiter,
450			'error' => $errorMessage,
451			];
452
453			if ($errorMessage === 0) $this->ActivityManager->triggerEvent($dataset, ActivityManager::OBJECT_DATA, ActivityManager::SUBJECT_DATA_ADD_IMPORT);
454			return $result;
455			} else {
456			return false;
457			}
458			}
459
460			/**
461			* Import data into dataset from an internal or external file
462			*
463			* @NoAdminRequired
464			* @param int $objectId
465			* @param $path
466			* @param bool $isDataset
467			* @return array\|false
468			* @throws \OCP\DB\Exception
469			*/
470			public function importFile(int $objectId, $path, bool $isDataset)
471			{
472			$dataset = $this->getDatasetId($objectId, $isDataset);
473			if ($dataset != '') {
474			$insert = $update = 0;
475			$reportMetadata = array();
476			$reportMetadata['link'] = $path;
477			$reportMetadata['user_id'] = $this->userId;
478			$result = $this->DatasourceController->read(DatasourceController::DATASET_TYPE_FILE, $reportMetadata);
479
480			if ($result['error'] === 0) {
481			foreach ($result['data'] as &$row) {
482			$action = $this->StorageService->update($dataset, $row[0], $row[1], $row[2]);
483			$insert = $insert + $action['insert'];
484			$update = $update + $action['update'];
485			}
486			}
487
488			$result = [
489			'insert' => $insert,
490			'update' => $update,
491			'error' => $result['error'],
492			];
493
494			if ($result['error'] === 0) $this->ActivityManager->triggerEvent($dataset, ActivityManager::OBJECT_DATA, ActivityManager::SUBJECT_DATA_ADD_IMPORT);
495			return $result;
496			} else {
497			return false;
498			}
499			}
500
501			private function getDatasetId($objectId, bool $isDataset)
502			{
503			if ($isDataset) {
504			$dataset = $objectId;
505			} else {
506			$reportMetadata = $this->ReportService->read($objectId);
507			$dataset = (int)$reportMetadata['dataset'];
508			}
509			return $dataset;
510			}
511
512			private function detectDelimiter($data): string
513			{
514			$delimiters = ["\t", ";", "\|", ","];
515			$data_2 = array();
516			$delimiter = $delimiters[0];
517			foreach ($delimiters as $d) {
518			$firstRow = str_getcsv($data, "\n")[0];
519			$data_1 = str_getcsv($firstRow, $d);
520			if (sizeof($data_1) > sizeof($data_2)) {
521			$delimiter = $d;
522			$data_2 = $data_1;
523			}
524			}
525			return $delimiter;
526			}
527
528			private function floatvalue($val)
529			{
530			// if value is a 3 digit comma number with one leading zero like 0,111, it should not go through the 1000 separator removal
531			if (preg_match('/(?<=\b0)\,(?=\d{3}\b)/', $val) === 0 && preg_match('/(?<=\b0)\.(?=\d{3}\b)/', $val) === 0) {
532			// remove , as 1000 separator
533			$val = preg_replace('/(?<=\d)\,(?=\d{3}\b)/', '', $val);
534			// remove . as 1000 separator
535			$val = preg_replace('/(?<=\d)\.(?=\d{3}\b)/', '', $val);
536			}
537			// convert remaining comma to decimal point
538			$val = str_replace(",", ".", $val);
539			if (is_numeric($val)) {
540			return number_format(floatval($val), 2, '.', '');
541			} else {
542			return false;
543			}
544			}
545
546			}

Rello / analytics

Push — master ( 952c0a...1807d7 )

DataloadService::copy() A

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like