DataloadService::copy() - Code Metrics - Rello/analytics - Measure and Improve Code Quality continuously with Scrutinizer

DataloadService::copy() A
last analyzed 2024-09-15 18:42 UTC

↳ Parent: DataloadService

Complexity

Conditions	1
Paths	1

Size

Total Lines	3
Code Lines	1

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
cc	1
eloc	1
nc	1
nop	1
dl	0
loc	3
rs	10
c	0
b	0
f	0

<?php
/**
 * Analytics
 *
 * SPDX-FileCopyrightText: 2019-2022 Marcel Scherello
 * SPDX-License-Identifier: AGPL-3.0-or-later
 */

namespace OCA\Analytics\Service;

use Exception;
use OCA\Analytics\Activity\ActivityManager;
use OCA\Analytics\Controller\DatasourceController;
use OCA\Analytics\Notification\NotificationManager;
use OCA\Analytics\Db\DataloadMapper;
use OCP\AppFramework\Http\NotFoundResponse;
filter:
    dependency_paths: ["lib/*"]
use OCP\Files\NotFoundException;
filter:
    dependency_paths: ["lib/*"]
use OCP\IL10N;
filter:
    dependency_paths: ["lib/*"]
use Psr\Log\LoggerInterface;
filter:
    dependency_paths: ["lib/*"]

class DataloadService
{
    private $userId;
    private $logger;
    private $StorageService;
    private $DatasourceController;
    private $ActivityManager;
    private $ReportService;
    private $DatasetService;
    private $VariableService;
    private $l10n;
    private $DataloadMapper;
    private $NotificationManager;

    public function __construct(
        $userId,
        IL10N $l10n,
        LoggerInterface $logger,
        ActivityManager $ActivityManager,
        DatasourceController $DatasourceController,
        ReportService $ReportService,
        DatasetService $DatasetService,
        StorageService $StorageService,
        VariableService $VariableService,
        NotificationManager $NotificationManager,
        DataloadMapper $DataloadMapper
    )
    {
        $this->userId = $userId;
        $this->l10n = $l10n;
        $this->logger = $logger;
        $this->StorageService = $StorageService;
        $this->ActivityManager = $ActivityManager;
        $this->DatasourceController = $DatasourceController;
        $this->ReportService = $ReportService;
        $this->DatasetService = $DatasetService;
        $this->VariableService = $VariableService;
        $this->NotificationManager = $NotificationManager;
        $this->DataloadMapper = $DataloadMapper;
    }

    // Data loads
    // Data loads
    // Data loads

    /**
     * create a new data load
     *
     * @param $datasetId
     * @param $reportId
     * @param int $datasourceId
     * @return int
     * @throws \OCP\DB\Exception
     */
    public function create($datasetId, int $datasourceId)
    {
        return $this->DataloadMapper->create((int)$datasetId, $datasourceId);
    }

    /**
     * get all data loads for a dataset or report
     *
     * @param int $datasetId
     * @param $reportId
     * @return array
     */
    public function read($datasetId)
    {
        return $this->DataloadMapper->read((int)$datasetId);
    }

    /**
     * update data load
     *
     * @param int $dataloadId
     * @param $name
     * @param $option
     * @param $schedule
     * @return bool
     */
    public function update(int $dataloadId, $name, $option, $schedule)
    {
        $array = json_decode($option, true);
        foreach ($array as $key => $value) {
            $array[$key] = htmlspecialchars($value, ENT_NOQUOTES, 'UTF-8');
        }
        $option = json_encode($array);

        return $this->DataloadMapper->update($dataloadId, $name, $option, $schedule);
    }

    /**
     * copy a data load
     *
     * @param int $dataloadId
     * @return bool
     */
    public function copy(int $dataloadId)
    {
        return $this->DataloadMapper->copy($dataloadId);
    }

    /**
     * delete a data load
     *
     * @param int $dataloadId
     * @return bool
     */
    public function delete(int $dataloadId)
    {
        return $this->DataloadMapper->delete($dataloadId);
    }

    /**
     * execute all data loads depending on their schedule
     * Daily or Hourly
     *
     * @param $schedule
     * @return void
     * @throws Exception
     */
    public function executeBySchedule($schedule)
    {
        $schedules = $this->DataloadMapper->getDataloadBySchedule($schedule);
        foreach ($schedules as $dataload) {
            $result = $this->execute($dataload['id']);
            if ($result['error'] !== 0) {
                // if the data source produced an error, a notification needs to be triggered
                $dataset = $this->DatasetService->read($dataload['dataset']);
                $this->NotificationManager->triggerNotification(NotificationManager::DATALOAD_ERROR, $dataload['dataset'], $dataload['id'], ['dataloadName' => $dataload['name'], 'datasetName' => $dataset['name']], $dataload['user_id']);
            }
        }
    }

    /**
     * execute a data load from data source and store into dataset
     *
     * @param int $dataloadId
     * @return array
     * @throws Exception
     */
    public function execute(int $dataloadId)
    {
        $bulkSize = 500;
        $insert = $update = $error = $delete = $currentCount = 0;
        $bulkInsert = null;
        $aggregation = null;

        // get the data from the datasource
        $result = $this->getDataFromDatasource($dataloadId);

        // dont continue in case of datasource error
        if ($result['error'] !== 0) {
            return [
                'insert' => $insert,
                'update' => $update,
                'delete' => $delete,
                'error' => 1,
            ];
        }

        // get the meta data
        $dataloadMetadata = $this->DataloadMapper->getDataloadById($dataloadId);
        $option = json_decode($dataloadMetadata['option'], true);
        $datasetId = $dataloadMetadata['dataset'];

        // this is a deletion request. Just run the deletion and stop after that with a return.
        if ($dataloadMetadata['datasource'] === 0) {
            // deletion jobs are using the same dimension/option/value settings a report filter
            $filter = array();
            $filter['filteroptions'] = '{"filter":{"' . $option['filterDimension'] . '":{"option":"' . $option['filterOption'] . '","value":"' . $option['filterValue'] . '"}}}';
            // Text variables like %xx% could be in use here
            $filter = $this->VariableService->replaceFilterVariables($filter);

            $records = $this->StorageService->deleteWithFilter($dataloadMetadata['dataset'], json_decode($filter['filteroptions'], true));

            return [
                'insert' => $insert,
                'update' => $update,
                'delete' => $records,
                'error' => $error,
            ];
        }

        // "delete all date before loading" is true in the data source options
        // in this case, bulkInsert is additionally set to true. Then no further checks for existing records are needed
        // to reduce db selects
        if (isset($option['delete']) and $option['delete'] === 'true') {
            $this->StorageService->delete($datasetId, '*', '*', $dataloadMetadata['user_id']);
            $bulkInsert = true;
        }

        // if the data set has no data, it is the same as the delete all option
        // in this case, bulkInsert is set to true. Then no further checks for existing records are needed
        // to reduce db selects
        $numberOfRecords = $this->StorageService->getRecordCount($datasetId, $dataloadMetadata['user_id']);
        if ($numberOfRecords['count'] === 0) {
            $bulkInsert = true;
        }

        // Feature not yet available
        if (isset($option['aggregation']) and $option['aggregation'] !== 'overwrite') {
            $aggregation = $option['aggregation'];
        }

        // collect mass updates to reduce statements to the database
        $this->DataloadMapper->beginTransaction();
        foreach ($result['data'] as $row) {
            // only one column is not possible
            if (count($row) === 1) {
                $this->logger->info('loading data with only one column is not possible. This is a data load for the dataset: ' . $datasetId);
                $error = $error + 1;
                continue;
            }
            // if data source only delivers 2 columns, the value needs to be in the last one
            if (count($row) === 2) {
                $row[2] = $row[1];
                $row[1] = null;
            }

            $action = $this->StorageService->update($datasetId, $row[0], $row[1], $row[2], $dataloadMetadata['user_id'], $bulkInsert, $aggregation);
            $insert = $insert + $action['insert'];
            $update = $update + $action['update'];
            $error = $error + $action['error'];

            if ($currentCount % $bulkSize === 0) {
                $this->DataloadMapper->commit();
                $this->DataloadMapper->beginTransaction();
            }
            if ($action['error'] === 0) $currentCount++;
        }
        $this->DataloadMapper->commit();

        $result = [
            'insert' => $insert,
            'update' => $update,
            'delete' => $delete,
            'error' => $error,
        ];

		// Update the Context Chat backend
		$this->DatasetService->provider($datasetId);

        //$this->ActivityManager->triggerEvent($datasetId, ActivityManager::OBJECT_DATA, ActivityManager::SUBJECT_DATA_ADD_DATALOAD, $dataloadMetadata['user_id']);
        return $result;
    }

    /**
     * get the data from datasource
     * to be used in simulation or execution
     *
     * @param int $dataloadId
     * @return array|NotFoundResponse
     * @throws NotFoundResponse
     * @throws \OCP\DB\Exception
     */
    public function getDataFromDatasource(int $dataloadId)
    {
        $dataloadMetadata = $this->DataloadMapper->getDataloadById($dataloadId);

        if (!empty($dataloadMetadata)) {

            if ($dataloadMetadata['datasource'] !== 0) {
                $dataloadMetadata['link'] = $dataloadMetadata['option']; //remap until data source table is renamed link=>option

                $result = $this->DatasourceController->read((int)$dataloadMetadata['datasource'], $dataloadMetadata);
                $result['datasetId'] = $dataloadMetadata['dataset'];
            } else {
                // this is a deletion request. Just run the simulation and return the possible row count in the expected result array
                $option = json_decode($dataloadMetadata['option'], true);

                // deletion jobs are using the same dimension/option/value settings a report filter
                $filter = array();
                $filter['filteroptions'] = '{"filter":{"' . $option['filterDimension'] . '":{"option":"' . $option['filterOption'] . '","value":"' . $option['filterValue'] . '"}}}';
                // Text variables like %xx% could be in use here
                $filter = $this->VariableService->replaceFilterVariables($filter);

                $result = [
                    'header' => '',
                    'dimensions' => '',
                    'data' => $this->StorageService->deleteWithFilterSimulate($dataloadMetadata['dataset'], json_decode($filter['filteroptions'], true)),
                    'error' => 0,
                ];
            }

            return $result;
        } else {
            return new NotFoundResponse();
        }
    }

    // Data Manipulation
    // Data Manipulation
    // Data Manipulation

    /**
     * update data from input form
     *
     * @NoAdminRequired
     * @param int $objectId
     * @param $dimension1
     * @param $dimension2
     * @param $value
     * @param bool $isDataset
     * @return array|false
     * @throws \OCP\DB\Exception
     */
    public function updateData(int $objectId, $dimension1, $dimension2, $value, bool $isDataset)
    {
        $datasetId = $this->getDatasetId($objectId, $isDataset);

        if ($datasetId != '') {
            $insert = $update = $errorMessage = 0;
            $action = array();
            $value = $this->floatvalue($value);
            if ($value === false) {
                $errorMessage = $this->l10n->t('3rd field must be a valid number');
            } else {
                $action = $this->StorageService->update($datasetId, $dimension1, $dimension2, $value);
                $insert = $insert + $action['insert'];
                $update = $update + $action['update'];
            }

            $result = [
                'insert' => $insert,
                'update' => $update,
                'error' => $errorMessage,
                'validate' => $action['validate'],
            ];

			// Update the Context Chat backend
			$this->DatasetService->provider($datasetId);

			//if ($errorMessage === 0) $this->ActivityManager->triggerEvent($dataset, ActivityManager::OBJECT_DATA, ActivityManager::SUBJECT_DATA_ADD);
            return $result;
        } else {
            return false;
        }
    }

    /**
     * Simulate delete data from input form
     *
     * @NoAdminRequired
     * @param int $objectId
     * @param $dimension1
     * @param $dimension2
     * @param bool $isDataset
     * @return array|false
     * @throws \OCP\DB\Exception
     */
    public function deleteDataSimulate(int $objectId, $dimension1, $dimension2, bool $isDataset)
    {
        $dataset = $this->getDatasetId($objectId, $isDataset);
        if ($dataset != '') {
            $result = $this->StorageService->deleteSimulate($dataset, $dimension1, $dimension2);
            return ['delete' => $result];
        } else {
            return false;
        }
    }

    /**
     * delete data from input form
     *
     * @NoAdminRequired
     * @param int $objectId
     * @param $dimension1
     * @param $dimension2
     * @param bool $isDataset
     * @return array|false
     * @throws \OCP\DB\Exception
     */
    public function deleteData(int $objectId, $dimension1, $dimension2, bool $isDataset)
    {
		$datasetId = $this->getDatasetId($objectId, $isDataset);
        if ($datasetId != '') {
            $result = $this->StorageService->delete($datasetId, $dimension1, $dimension2);

			// Update the Context Chat backend
			$this->DatasetService->provider($datasetId);

			return ['delete' => $result];
        } else {
            return false;
        }
    }

    /**
     * Import clipboard data
     *
     * @NoAdminRequired
     * @param int $objectId
     * @param $import
     * @param bool $isDataset
     * @return array|false
     * @throws \OCP\DB\Exception
     */
    public function importClipboard($objectId, $import, bool $isDataset)
    {
        $datasetId = $this->getDatasetId($objectId, $isDataset);
        if ($datasetId != '') {
            $insert = $update = $errorMessage = $errorCounter = 0;
            $delimiter = '';

            if ($import === '') {
                $errorMessage = $this->l10n->t('No data');
            } else {
                $delimiter = $this->detectDelimiter($import);
                $rows = str_getcsv($import, "\n");

                foreach ($rows as &$row) {
                    $row = str_getcsv($row, $delimiter);
                    $numberOfColumns = count($row);
                    // last column needs to be a float
                    $row[2] = $this->floatvalue($row[$numberOfColumns - 1]);
                    if ($row[2] === false) {
                        $errorCounter++;
                    } else {
                        if ($numberOfColumns < 3) $row[1] = null;
                        $action = $this->StorageService->update($datasetId, $row[0], $row[1], $row[2]);
                        $insert = $insert + $action['insert'];
                        $update = $update + $action['update'];
                    }
                    if ($errorCounter === 2) {
                        // first error is ignored; might be due to header row
                        $errorMessage = $this->l10n->t('Last field must be a valid number');
                        break;
                    }
                }
            }

            $result = [
                'insert' => $insert,
                'update' => $update,
                'delimiter' => $delimiter,
                'error' => $errorMessage,
            ];

			// Update the Context Chat backend
			$this->DatasetService->provider($datasetId);

			//if ($errorMessage === 0) $this->ActivityManager->triggerEvent($dataset, ActivityManager::OBJECT_DATA, ActivityManager::SUBJECT_DATA_ADD_IMPORT);
            return $result;
        } else {
            return false;
        }
    }

    /**
     * Import data into dataset from an internal or external file
     *
     * @NoAdminRequired
     * @param int $objectId
     * @param $path
     * @param bool $isDataset
     * @return array|false
     * @throws \OCP\DB\Exception
     */
    public function importFile(int $objectId, $path, bool $isDataset)
    {
        $datasetId = $this->getDatasetId($objectId, $isDataset);
        if ($datasetId != '') {
            $insert = $update = 0;
            $reportMetadata = array();
            $reportMetadata['link'] = $path;
            $reportMetadata['user_id'] = $this->userId;
            $result = $this->DatasourceController->read(DatasourceController::DATASET_TYPE_FILE, $reportMetadata);

            if ($result['error'] === 0) {
                foreach ($result['data'] as &$row) {
                    $action = $this->StorageService->update($datasetId, $row[0], $row[1], $row[2]);
                    $insert = $insert + $action['insert'];
                    $update = $update + $action['update'];
                }
            }

            $result = [
                'insert' => $insert,
                'update' => $update,
                'error' => $result['error'],
            ];

			// Update the Context Chat backend
			$this->DatasetService->provider($datasetId);

			//if ($result['error'] === 0) $this->ActivityManager->triggerEvent($dataset, ActivityManager::OBJECT_DATA, ActivityManager::SUBJECT_DATA_ADD_IMPORT);
            return $result;
        } else {
            return false;
        }
    }

    private function getDatasetId($objectId, bool $isDataset)
    {
        if ($isDataset) {
            $dataset = $objectId;
        } else {
            $reportMetadata = $this->ReportService->read($objectId);
            $dataset = (int)$reportMetadata['dataset'];
        }
        return $dataset;
    }

    private function detectDelimiter($data): string
    {
        $delimiters = ["\t", ";", "|", ","];
        $data_2 = array();
        $delimiter = $delimiters[0];
        foreach ($delimiters as $d) {
            $firstRow = str_getcsv($data, "\n")[0];
            $data_1 = str_getcsv($firstRow, $d);
            if (sizeof($data_1) > sizeof($data_2)) {
                $delimiter = $d;
                $data_2 = $data_1;
            }
        }
        return $delimiter;
    }

    private function floatvalue($val)
    {
        // if value is a 3 digit comma number with one leading zero like 0,111, it should not go through the 1000 separator removal
        if (preg_match('/(?<=\b0)\,(?=\d{3}\b)/', $val) === 0 && preg_match('/(?<=\b0)\.(?=\d{3}\b)/', $val) === 0) {
            // remove , as 1000 separator
            $val = preg_replace('/(?<=\d)\,(?=\d{3}\b)/', '', $val);
            // remove . as 1000 separator
            $val = preg_replace('/(?<=\d)\.(?=\d{3}\b)/', '', $val);
        }
        // convert remaining comma to decimal point
        $val = str_replace(",", ".", $val);
        if (is_numeric($val)) {
            return number_format(floatval($val), 2, '.', '');
        } else {
            return false;
        }
    }

}

1			<?php
2			/**
3			* Analytics
4			*
5			* SPDX-FileCopyrightText: 2019-2022 Marcel Scherello
6			* SPDX-License-Identifier: AGPL-3.0-or-later
7			*/
8
9			namespace OCA\Analytics\Service;
10
11			use Exception;
12			use OCA\Analytics\Activity\ActivityManager;
13			use OCA\Analytics\Controller\DatasourceController;
14			use OCA\Analytics\Notification\NotificationManager;
15			use OCA\Analytics\Db\DataloadMapper;
16			use OCP\AppFramework\Http\NotFoundResponse;
			0 ignored issues – show Bug introduced 2023-08-25 06:43 UTC by Report Bug Copy Issue Report The type `OCP\AppFramework\Http\NotFoundResponse` was not found. Maybe you did not declare it correctly or list all dependencies? The issue could also be caused by a filter entry in the build configuration. If the path has been excluded in your configuration, e.g. `excluded_paths: ["lib/"]`, you can move it to the dependency path list as follows: filter: dependency_paths: ["lib/"] For further information see https://scrutinizer-ci.com/docs/tools/php/php-scrutinizer/#list-dependency-paths Loading history...
17			use OCP\Files\NotFoundException;
			0 ignored issues – show Bug introduced 2023-08-25 06:43 UTC by Report Bug Copy Issue Report The type `OCP\Files\NotFoundException` was not found. Maybe you did not declare it correctly or list all dependencies? The issue could also be caused by a filter entry in the build configuration. If the path has been excluded in your configuration, e.g. `excluded_paths: ["lib/"]`, you can move it to the dependency path list as follows: filter: dependency_paths: ["lib/"] For further information see https://scrutinizer-ci.com/docs/tools/php/php-scrutinizer/#list-dependency-paths Loading history...
18			use OCP\IL10N;
			0 ignored issues – show Bug introduced 2023-08-25 06:43 UTC by Report Bug Copy Issue Report The type `OCP\IL10N` was not found. Maybe you did not declare it correctly or list all dependencies? The issue could also be caused by a filter entry in the build configuration. If the path has been excluded in your configuration, e.g. `excluded_paths: ["lib/"]`, you can move it to the dependency path list as follows: filter: dependency_paths: ["lib/"] For further information see https://scrutinizer-ci.com/docs/tools/php/php-scrutinizer/#list-dependency-paths Loading history...
19			use Psr\Log\LoggerInterface;
			0 ignored issues – show Bug introduced 2023-08-25 06:43 UTC by Report Bug Copy Issue Report The type `Psr\Log\LoggerInterface` was not found. Maybe you did not declare it correctly or list all dependencies? The issue could also be caused by a filter entry in the build configuration. If the path has been excluded in your configuration, e.g. `excluded_paths: ["lib/"]`, you can move it to the dependency path list as follows: filter: dependency_paths: ["lib/"] For further information see https://scrutinizer-ci.com/docs/tools/php/php-scrutinizer/#list-dependency-paths Loading history...
20
21			class DataloadService
22			{
23			private $userId;
24			private $logger;
25			private $StorageService;
26			private $DatasourceController;
27			private $ActivityManager;
28			private $ReportService;
29			private $DatasetService;
30			private $VariableService;
31			private $l10n;
32			private $DataloadMapper;
33			private $NotificationManager;
34
35			public function __construct(
36			$userId,
37			IL10N $l10n,
38			LoggerInterface $logger,
39			ActivityManager $ActivityManager,
40			DatasourceController $DatasourceController,
41			ReportService $ReportService,
42			DatasetService $DatasetService,
43			StorageService $StorageService,
44			VariableService $VariableService,
45			NotificationManager $NotificationManager,
46			DataloadMapper $DataloadMapper
47			)
48			{
49			$this->userId = $userId;
50			$this->l10n = $l10n;
51			$this->logger = $logger;
52			$this->StorageService = $StorageService;
53			$this->ActivityManager = $ActivityManager;
54			$this->DatasourceController = $DatasourceController;
55			$this->ReportService = $ReportService;
56			$this->DatasetService = $DatasetService;
57			$this->VariableService = $VariableService;
58			$this->NotificationManager = $NotificationManager;
59			$this->DataloadMapper = $DataloadMapper;
60			}
61
62			// Data loads
63			// Data loads
64			// Data loads
65
66			/**
67			* create a new data load
68			*
69			* @param $datasetId
70			* @param $reportId
71			* @param int $datasourceId
72			* @return int
73			* @throws \OCP\DB\Exception
74			*/
75			public function create($datasetId, int $datasourceId)
76			{
77			return $this->DataloadMapper->create((int)$datasetId, $datasourceId);
78			}
79
80			/**
81			* get all data loads for a dataset or report
82			*
83			* @param int $datasetId
84			* @param $reportId
85			* @return array
86			*/
87			public function read($datasetId)
88			{
89			return $this->DataloadMapper->read((int)$datasetId);
90			}
91
92			/**
93			* update data load
94			*
95			* @param int $dataloadId
96			* @param $name
97			* @param $option
98			* @param $schedule
99			* @return bool
100			*/
101			public function update(int $dataloadId, $name, $option, $schedule)
102			{
103			$array = json_decode($option, true);
104			foreach ($array as $key => $value) {
105			$array[$key] = htmlspecialchars($value, ENT_NOQUOTES, 'UTF-8');
106			}
107			$option = json_encode($array);
108
109			return $this->DataloadMapper->update($dataloadId, $name, $option, $schedule);
110			}
111
112			/**
113			* copy a data load
114			*
115			* @param int $dataloadId
116			* @return bool
117			*/
118			public function copy(int $dataloadId)
119			{
120			return $this->DataloadMapper->copy($dataloadId);
121			}
122
123			/**
124			* delete a data load
125			*
126			* @param int $dataloadId
127			* @return bool
128			*/
129			public function delete(int $dataloadId)
130			{
131			return $this->DataloadMapper->delete($dataloadId);
132			}
133
134			/**
135			* execute all data loads depending on their schedule
136			* Daily or Hourly
137			*
138			* @param $schedule
139			* @return void
140			* @throws Exception
141			*/
142			public function executeBySchedule($schedule)
143			{
144			$schedules = $this->DataloadMapper->getDataloadBySchedule($schedule);
145			foreach ($schedules as $dataload) {
146			$result = $this->execute($dataload['id']);
147			if ($result['error'] !== 0) {
148			// if the data source produced an error, a notification needs to be triggered
149			$dataset = $this->DatasetService->read($dataload['dataset']);
150			$this->NotificationManager->triggerNotification(NotificationManager::DATALOAD_ERROR, $dataload['dataset'], $dataload['id'], ['dataloadName' => $dataload['name'], 'datasetName' => $dataset['name']], $dataload['user_id']);
151			}
152			}
153			}
154
155			/**
156			* execute a data load from data source and store into dataset
157			*
158			* @param int $dataloadId
159			* @return array
160			* @throws Exception
161			*/
162			public function execute(int $dataloadId)
163			{
164			$bulkSize = 500;
165			$insert = $update = $error = $delete = $currentCount = 0;
166			$bulkInsert = null;
167			$aggregation = null;
168
169			// get the data from the datasource
170			$result = $this->getDataFromDatasource($dataloadId);
171
172			// dont continue in case of datasource error
173			if ($result['error'] !== 0) {
174			return [
175			'insert' => $insert,
176			'update' => $update,
177			'delete' => $delete,
178			'error' => 1,
179			];
180			}
181
182			// get the meta data
183			$dataloadMetadata = $this->DataloadMapper->getDataloadById($dataloadId);
184			$option = json_decode($dataloadMetadata['option'], true);
185			$datasetId = $dataloadMetadata['dataset'];
186
187			// this is a deletion request. Just run the deletion and stop after that with a return.
188			if ($dataloadMetadata['datasource'] === 0) {
189			// deletion jobs are using the same dimension/option/value settings a report filter
190			$filter = array();
191			$filter['filteroptions'] = '{"filter":{"' . $option['filterDimension'] . '":{"option":"' . $option['filterOption'] . '","value":"' . $option['filterValue'] . '"}}}';
192			// Text variables like %xx% could be in use here
193			$filter = $this->VariableService->replaceFilterVariables($filter);
194
195			$records = $this->StorageService->deleteWithFilter($dataloadMetadata['dataset'], json_decode($filter['filteroptions'], true));
196
197			return [
198			'insert' => $insert,
199			'update' => $update,
200			'delete' => $records,
201			'error' => $error,
202			];
203			}
204
205			// "delete all date before loading" is true in the data source options
206			// in this case, bulkInsert is additionally set to true. Then no further checks for existing records are needed
207			// to reduce db selects
208			if (isset($option['delete']) and $option['delete'] === 'true') {
209			$this->StorageService->delete($datasetId, '', '', $dataloadMetadata['user_id']);
210			$bulkInsert = true;
211			}
212
213			// if the data set has no data, it is the same as the delete all option
214			// in this case, bulkInsert is set to true. Then no further checks for existing records are needed
215			// to reduce db selects
216			$numberOfRecords = $this->StorageService->getRecordCount($datasetId, $dataloadMetadata['user_id']);
217			if ($numberOfRecords['count'] === 0) {
218			$bulkInsert = true;
219			}
220
221			// Feature not yet available
222			if (isset($option['aggregation']) and $option['aggregation'] !== 'overwrite') {
223			$aggregation = $option['aggregation'];
224			}
225
226			// collect mass updates to reduce statements to the database
227			$this->DataloadMapper->beginTransaction();
228			foreach ($result['data'] as $row) {
229			// only one column is not possible
230			if (count($row) === 1) {
231			$this->logger->info('loading data with only one column is not possible. This is a data load for the dataset: ' . $datasetId);
232			$error = $error + 1;
233			continue;
234			}
235			// if data source only delivers 2 columns, the value needs to be in the last one
236			if (count($row) === 2) {
237			$row[2] = $row[1];
238			$row[1] = null;
239			}
240
241			$action = $this->StorageService->update($datasetId, $row[0], $row[1], $row[2], $dataloadMetadata['user_id'], $bulkInsert, $aggregation);
242			$insert = $insert + $action['insert'];
243			$update = $update + $action['update'];
244			$error = $error + $action['error'];
245
246			if ($currentCount % $bulkSize === 0) {
247			$this->DataloadMapper->commit();
248			$this->DataloadMapper->beginTransaction();
249			}
250			if ($action['error'] === 0) $currentCount++;
251			}
252			$this->DataloadMapper->commit();
253
254			$result = [
255			'insert' => $insert,
256			'update' => $update,
257			'delete' => $delete,
258			'error' => $error,
259			];
260
261			// Update the Context Chat backend
262			$this->DatasetService->provider($datasetId);
263
264			//$this->ActivityManager->triggerEvent($datasetId, ActivityManager::OBJECT_DATA, ActivityManager::SUBJECT_DATA_ADD_DATALOAD, $dataloadMetadata['user_id']);
265			return $result;
266			}
267
268			/**
269			* get the data from datasource
270			* to be used in simulation or execution
271			*
272			* @param int $dataloadId
273			* @return array\|NotFoundResponse
274			* @throws NotFoundResponse
275			* @throws \OCP\DB\Exception
276			*/
277			public function getDataFromDatasource(int $dataloadId)
278			{
279			$dataloadMetadata = $this->DataloadMapper->getDataloadById($dataloadId);
280
281			if (!empty($dataloadMetadata)) {
282
283			if ($dataloadMetadata['datasource'] !== 0) {
284			$dataloadMetadata['link'] = $dataloadMetadata['option']; //remap until data source table is renamed link=>option
285
286			$result = $this->DatasourceController->read((int)$dataloadMetadata['datasource'], $dataloadMetadata);
287			$result['datasetId'] = $dataloadMetadata['dataset'];
288			} else {
289			// this is a deletion request. Just run the simulation and return the possible row count in the expected result array
290			$option = json_decode($dataloadMetadata['option'], true);
291
292			// deletion jobs are using the same dimension/option/value settings a report filter
293			$filter = array();
294			$filter['filteroptions'] = '{"filter":{"' . $option['filterDimension'] . '":{"option":"' . $option['filterOption'] . '","value":"' . $option['filterValue'] . '"}}}';
295			// Text variables like %xx% could be in use here
296			$filter = $this->VariableService->replaceFilterVariables($filter);
297
298			$result = [
299			'header' => '',
300			'dimensions' => '',
301			'data' => $this->StorageService->deleteWithFilterSimulate($dataloadMetadata['dataset'], json_decode($filter['filteroptions'], true)),
302			'error' => 0,
303			];
304			}
305
306			return $result;
307			} else {
308			return new NotFoundResponse();
309			}
310			}
311
312			// Data Manipulation
313			// Data Manipulation
314			// Data Manipulation
315
316			/**
317			* update data from input form
318			*
319			* @NoAdminRequired
320			* @param int $objectId
321			* @param $dimension1
322			* @param $dimension2
323			* @param $value
324			* @param bool $isDataset
325			* @return array\|false
326			* @throws \OCP\DB\Exception
327			*/
328			public function updateData(int $objectId, $dimension1, $dimension2, $value, bool $isDataset)
329			{
330			$datasetId = $this->getDatasetId($objectId, $isDataset);
331
332			if ($datasetId != '') {
333			$insert = $update = $errorMessage = 0;
334			$action = array();
335			$value = $this->floatvalue($value);
336			if ($value === false) {
337			$errorMessage = $this->l10n->t('3rd field must be a valid number');
338			} else {
339			$action = $this->StorageService->update($datasetId, $dimension1, $dimension2, $value);
340			$insert = $insert + $action['insert'];
341			$update = $update + $action['update'];
342			}
343
344			$result = [
345			'insert' => $insert,
346			'update' => $update,
347			'error' => $errorMessage,
348			'validate' => $action['validate'],
349			];
350
351			// Update the Context Chat backend
352			$this->DatasetService->provider($datasetId);
353
354			//if ($errorMessage === 0) $this->ActivityManager->triggerEvent($dataset, ActivityManager::OBJECT_DATA, ActivityManager::SUBJECT_DATA_ADD);
355			return $result;
356			} else {
357			return false;
358			}
359			}
360
361			/**
362			* Simulate delete data from input form
363			*
364			* @NoAdminRequired
365			* @param int $objectId
366			* @param $dimension1
367			* @param $dimension2
368			* @param bool $isDataset
369			* @return array\|false
370			* @throws \OCP\DB\Exception
371			*/
372			public function deleteDataSimulate(int $objectId, $dimension1, $dimension2, bool $isDataset)
373			{
374			$dataset = $this->getDatasetId($objectId, $isDataset);
375			if ($dataset != '') {
376			$result = $this->StorageService->deleteSimulate($dataset, $dimension1, $dimension2);
377			return ['delete' => $result];
378			} else {
379			return false;
380			}
381			}
382
383			/**
384			* delete data from input form
385			*
386			* @NoAdminRequired
387			* @param int $objectId
388			* @param $dimension1
389			* @param $dimension2
390			* @param bool $isDataset
391			* @return array\|false
392			* @throws \OCP\DB\Exception
393			*/
394			public function deleteData(int $objectId, $dimension1, $dimension2, bool $isDataset)
395			{
396			$datasetId = $this->getDatasetId($objectId, $isDataset);
397			if ($datasetId != '') {
398			$result = $this->StorageService->delete($datasetId, $dimension1, $dimension2);
399
400			// Update the Context Chat backend
401			$this->DatasetService->provider($datasetId);
402
403			return ['delete' => $result];
404			} else {
405			return false;
406			}
407			}
408
409			/**
410			* Import clipboard data
411			*
412			* @NoAdminRequired
413			* @param int $objectId
414			* @param $import
415			* @param bool $isDataset
416			* @return array\|false
417			* @throws \OCP\DB\Exception
418			*/
419			public function importClipboard($objectId, $import, bool $isDataset)
420			{
421			$datasetId = $this->getDatasetId($objectId, $isDataset);
422			if ($datasetId != '') {
423			$insert = $update = $errorMessage = $errorCounter = 0;
424			$delimiter = '';
425
426			if ($import === '') {
427			$errorMessage = $this->l10n->t('No data');
428			} else {
429			$delimiter = $this->detectDelimiter($import);
430			$rows = str_getcsv($import, "\n");
431
432			foreach ($rows as &$row) {
433			$row = str_getcsv($row, $delimiter);
434			$numberOfColumns = count($row);
435			// last column needs to be a float
436			$row[2] = $this->floatvalue($row[$numberOfColumns - 1]);
437			if ($row[2] === false) {
438			$errorCounter++;
439			} else {
440			if ($numberOfColumns < 3) $row[1] = null;
441			$action = $this->StorageService->update($datasetId, $row[0], $row[1], $row[2]);
442			$insert = $insert + $action['insert'];
443			$update = $update + $action['update'];
444			}
445			if ($errorCounter === 2) {
446			// first error is ignored; might be due to header row
447			$errorMessage = $this->l10n->t('Last field must be a valid number');
448			break;
449			}
450			}
451			}
452
453			$result = [
454			'insert' => $insert,
455			'update' => $update,
456			'delimiter' => $delimiter,
457			'error' => $errorMessage,
458			];
459
460			// Update the Context Chat backend
461			$this->DatasetService->provider($datasetId);
462
463			//if ($errorMessage === 0) $this->ActivityManager->triggerEvent($dataset, ActivityManager::OBJECT_DATA, ActivityManager::SUBJECT_DATA_ADD_IMPORT);
464			return $result;
465			} else {
466			return false;
467			}
468			}
469
470			/**
471			* Import data into dataset from an internal or external file
472			*
473			* @NoAdminRequired
474			* @param int $objectId
475			* @param $path
476			* @param bool $isDataset
477			* @return array\|false
478			* @throws \OCP\DB\Exception
479			*/
480			public function importFile(int $objectId, $path, bool $isDataset)
481			{
482			$datasetId = $this->getDatasetId($objectId, $isDataset);
483			if ($datasetId != '') {
484			$insert = $update = 0;
485			$reportMetadata = array();
486			$reportMetadata['link'] = $path;
487			$reportMetadata['user_id'] = $this->userId;
488			$result = $this->DatasourceController->read(DatasourceController::DATASET_TYPE_FILE, $reportMetadata);
489
490			if ($result['error'] === 0) {
491			foreach ($result['data'] as &$row) {
492			$action = $this->StorageService->update($datasetId, $row[0], $row[1], $row[2]);
493			$insert = $insert + $action['insert'];
494			$update = $update + $action['update'];
495			}
496			}
497
498			$result = [
499			'insert' => $insert,
500			'update' => $update,
501			'error' => $result['error'],
502			];
503
504			// Update the Context Chat backend
505			$this->DatasetService->provider($datasetId);
506
507			//if ($result['error'] === 0) $this->ActivityManager->triggerEvent($dataset, ActivityManager::OBJECT_DATA, ActivityManager::SUBJECT_DATA_ADD_IMPORT);
508			return $result;
509			} else {
510			return false;
511			}
512			}
513
514			private function getDatasetId($objectId, bool $isDataset)
515			{
516			if ($isDataset) {
517			$dataset = $objectId;
518			} else {
519			$reportMetadata = $this->ReportService->read($objectId);
520			$dataset = (int)$reportMetadata['dataset'];
521			}
522			return $dataset;
523			}
524
525			private function detectDelimiter($data): string
526			{
527			$delimiters = ["\t", ";", "\|", ","];
528			$data_2 = array();
529			$delimiter = $delimiters[0];
530			foreach ($delimiters as $d) {
531			$firstRow = str_getcsv($data, "\n")[0];
532			$data_1 = str_getcsv($firstRow, $d);
533			if (sizeof($data_1) > sizeof($data_2)) {
534			$delimiter = $d;
535			$data_2 = $data_1;
536			}
537			}
538			return $delimiter;
539			}
540
541			private function floatvalue($val)
542			{
543			// if value is a 3 digit comma number with one leading zero like 0,111, it should not go through the 1000 separator removal
544			if (preg_match('/(?<=\b0)\,(?=\d{3}\b)/', $val) === 0 && preg_match('/(?<=\b0)\.(?=\d{3}\b)/', $val) === 0) {
545			// remove , as 1000 separator
546			$val = preg_replace('/(?<=\d)\,(?=\d{3}\b)/', '', $val);
547			// remove . as 1000 separator
548			$val = preg_replace('/(?<=\d)\.(?=\d{3}\b)/', '', $val);
549			}
550			// convert remaining comma to decimal point
551			$val = str_replace(",", ".", $val);
552			if (is_numeric($val)) {
553			return number_format(floatval($val), 2, '.', '');
554			} else {
555			return false;
556			}
557			}
558
559			}

Rello / analytics

DataloadService::copy() A last analyzed 2024-09-15 18:42 UTC

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like

DataloadService::copy() A
last analyzed 2024-09-15 18:42 UTC