DataloadService::executeBySchedule() - Code Metrics - Inspection of "Preparation for key figure aggregation mode in dat..." - Rello/analytics - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Push — master ( 1876ce...d1e05a )

by Marcel

created 2023-01-31 16:26 UTC

DataloadService::executeBySchedule() A

↳ Parent: DataloadService

Complexity

Conditions	3
Paths	3

Size

Total Lines	9
Code Lines	6

Duplication

Lines	0
Ratio	0 %

Importance

Changes	1
Bugs	0	Features	0

Metric	Value
cc	3
eloc	6
nc	3
nop	1
dl	0
loc	9
rs	10
c	1
b	0
f	0

<?php
/**
 * Analytics
 *
 * This file is licensed under the Affero General Public License version 3 or
 * later. See the LICENSE.md file.
 *
 * @author Marcel Scherello <[email protected]>
 * @copyright 2019-2022 Marcel Scherello
 */

namespace OCA\Analytics\Service;

use Exception;
use OCA\Analytics\Activity\ActivityManager;
use OCA\Analytics\Controller\DatasourceController;
use OCA\Analytics\Notification\NotificationManager;
use OCA\Analytics\Db\DataloadMapper;
use OCP\AppFramework\Http\NotFoundResponse;
use OCP\Files\NotFoundException;
use OCP\IL10N;
use Psr\Log\LoggerInterface;

class DataloadService
{
    private $userId;
    private $logger;
    private $StorageService;
    private $DatasourceController;
    private $ActivityManager;
    private $ReportService;
    private $DatasetService;
    private $VariableService;
    private $l10n;
    private $DataloadMapper;
    private $NotificationManager;

    public function __construct(
        $userId,
        IL10N $l10n,
        LoggerInterface $logger,
        ActivityManager $ActivityManager,
        DatasourceController $DatasourceController,
        ReportService $ReportService,
        DatasetService $DatasetService,
        StorageService $StorageService,
        VariableService $VariableService,
        NotificationManager $NotificationManager,
        DataloadMapper $DataloadMapper
    )
    {
        $this->userId = $userId;
        $this->l10n = $l10n;
        $this->logger = $logger;
        $this->StorageService = $StorageService;
        $this->ActivityManager = $ActivityManager;
        $this->DatasourceController = $DatasourceController;
        $this->ReportService = $ReportService;
        $this->DatasetService = $DatasetService;
        $this->VariableService = $VariableService;
        $this->NotificationManager = $NotificationManager;
        $this->DataloadMapper = $DataloadMapper;
    }

    // Data loads
    // Data loads
    // Data loads

    /**
     * create a new data load
     *
     * @param $datasetId
     * @param $reportId
     * @param int $datasourceId
     * @return int
     * @throws \OCP\DB\Exception
     */
    public function create($datasetId, int $datasourceId)
    {
        return $this->DataloadMapper->create((int)$datasetId, $datasourceId);
    }

    /**
     * get all data loads for a dataset or report
     *
     * @param int $datasetId
     * @param $reportId
     * @return array
     */
    public function read($datasetId)
    {
        return $this->DataloadMapper->read((int)$datasetId);
    }

    /**
     * update data load
     *
     * @param int $dataloadId
     * @param $name
     * @param $option
     * @param $schedule
     * @return bool
     */
    public function update(int $dataloadId, $name, $option, $schedule)
    {
        $array = json_decode($option, true);
        foreach ($array as $key => $value) {
            $array[$key] = htmlspecialchars($value, ENT_NOQUOTES, 'UTF-8');
        }
        $option = json_encode($array);

        return $this->DataloadMapper->update($dataloadId, $name, $option, $schedule);
    }

    /**
     * delete a data load
     *
     * @param int $dataloadId
     * @return bool
     */
    public function delete(int $dataloadId)
    {
        return $this->DataloadMapper->delete($dataloadId);
    }

    /**
     * execute all data loads depending on their schedule
     * Daily or Hourly
     *
     * @param $schedule
     * @return void
     * @throws Exception
     */
    public function executeBySchedule($schedule)
    {
        $schedules = $this->DataloadMapper->getDataloadBySchedule($schedule);
        foreach ($schedules as $dataload) {
            $result = $this->execute($dataload['id']);
            if ($result['error'] !== 0) {
                // if the data source produced an error, a notification needs to be triggered
                $dataset = $this->DatasetService->read($dataload['dataset']);
                $this->NotificationManager->triggerNotification(NotificationManager::DATALOAD_ERROR, $dataload['dataset'], $dataload['id'], ['dataloadName' => $dataload['name'], 'datasetName' => $dataset['name']], $dataload['user_id']);
            }
        }
    }

    /**
     * execute a data load from data source and store into dataset
     *
     * @param int $dataloadId
     * @return array
     * @throws Exception
     */
    public function execute(int $dataloadId)
    {
        $bulkSize = 500;
        $insert = $update = $error = $delete = 0;
        $bulkInsert = null;
        $aggregation = null;

        // get the meta data
        $dataloadMetadata = $this->DataloadMapper->getDataloadById($dataloadId);
        $option = json_decode($dataloadMetadata['option'], true);

        // get the data from the datasource
        $result = $this->getDataFromDatasource($dataloadId);
        $datasetId = $result['datasetId'];

        // dont continue in case of datasource error
        if ($result['error'] !== 0) {
            return [
                'insert' => $insert,
                'update' => $update,
                'delete' => $delete,
                'error' => 1,
            ];
        }

        // this is a deletion request. Just run the deletion and stop after that with a return.
        if ($dataloadMetadata['datasource'] === 0) {
            $option = json_decode($dataloadMetadata['option'], true);

            // deletion jobs are using the same dimension/option/value settings a report filter
            $filter['filteroptions'] = '{"filter":{"' . $option['filterDimension'] . '":{"option":"' . $option['filterOption'] . '","value":"' . $option['filterValue'] . '"}}}';

            // Text variables like %xx% could be in use here
            $filter = $this->VariableService->replaceFilterVariables($filter);

            $this->StorageService->deleteWithFilterSimulate($dataloadMetadata['dataset'], json_decode($filter['filteroptions'], true));

            return [
                'insert' => $insert,
                'update' => $update,
                // use the existing row count from the deletion simulation which was run during getDataFromDatasource
                'delete' => $result['data']['count'],
                'error' => $error,
            ];
        }

        // "delete all date before loading" is true in the data source options
        // in this case, bulkInsert is set to true. Then no further checks for existing records are needed
        // reduce db selects
        if (isset($option['delete']) and $option['delete'] === 'true') {
            $this->StorageService->delete($datasetId, '*', '*', $dataloadMetadata['user_id']);
            $bulkInsert = true;
        }

        // if the data set has no data, it is the same as the delete all option
        // in this case, bulkInsert is set to true. Then no further checks for existing records are needed
        // reduce db selects
        $numberOfRecords = $this->StorageService->getRecordCount($datasetId);
        if ($numberOfRecords['count'] === 0) {
            $this->logger->info('Empty dataset => enabling bulk load');
            $bulkInsert = true;
        }

        // Feature not yet available
        if (isset($option['aggregation']) and $option['aggregation'] !== 'overwrite') {
            $aggregation = $option['aggregation'];
        }

        // collect mass updates to reduce statements to the database
        $this->DataloadMapper->beginTransaction();
        $currentCount = 0;
        foreach ($result['data'] as $row) {
            if (count($row) === 1) {
                // only one column is not possible
                $this->logger->info('loading data with only one column is not possible. This is a data load for the dataset: ' . $datasetId);
                $error = $error + 1;
                continue;
            }
            if (count($row) === 2) {
                // if data source only delivers 2 columns, the value needs to be in the last one
                $row[2] = $row[1];
                $row[1] = null;
            }
            $action = $this->StorageService->update($datasetId, $row[0], $row[1], $row[2], $dataloadMetadata['user_id'], $bulkInsert, $aggregation);
            $insert = $insert + $action['insert'];
            $update = $update + $action['update'];
            $error = $error + $action['error'];

            if ($currentCount % $bulkSize === 0) {
                $this->DataloadMapper->commit();
                $this->DataloadMapper->beginTransaction();
            }
            if ($action['error'] === 0) $currentCount++;
        }
        $this->DataloadMapper->commit();

        $result = [
            'insert' => $insert,
            'update' => $update,
            'delete' => $delete,
            'error' => $error,
        ];

        $this->ActivityManager->triggerEvent($datasetId, ActivityManager::OBJECT_DATA, ActivityManager::SUBJECT_DATA_ADD_DATALOAD, $dataloadMetadata['user_id']);
        return $result;
    }

    /**
     * get the data from datasource
     * to be used in simulation or execution
     *
     * @param int $dataloadId
     * @return array|NotFoundResponse
     * @throws NotFoundResponse|NotFoundException
     * @throws \OCP\DB\Exception
     */
    public function getDataFromDatasource(int $dataloadId)
    {
        $dataloadMetadata = $this->DataloadMapper->getDataloadById($dataloadId);

        if (!empty($dataloadMetadata)) {

            if ($dataloadMetadata['datasource'] !== 0) {
                $dataloadMetadata['link'] = $dataloadMetadata['option']; //remap until data source table is renamed link=>option
                $result = $this->DatasourceController->read((int)$dataloadMetadata['datasource'], $dataloadMetadata);
                $result['datasetId'] = $dataloadMetadata['dataset'];
            } else {
                // this is a deletion request. Just run the simulation and return the possible row count in the expected result array
                $option = json_decode($dataloadMetadata['option'], true);

                // deletion jobs are using the same dimension/option/value settings a report filter
                $filter['filteroptions'] = '{"filter":{"' . $option['filterDimension'] . '":{"option":"' . $option['filterOption'] . '","value":"' . $option['filterValue'] . '"}}}';

                // Text variables like %xx% could be in use here
                $filter = $this->VariableService->replaceFilterVariables($filter);

                $result = [
                    'header' => '',
                    'dimensions' => '',
                    'data' => $this->StorageService->deleteWithFilterSimulate($dataloadMetadata['dataset'], json_decode($filter['filteroptions'], true)),
                    'error' => 0,
                ];
            }

            return $result;

        } else {
            return new NotFoundResponse();
        }
    }

    // Data Manipulation
    // Data Manipulation
    // Data Manipulation

    /**
     * update data from input form
     *
     * @NoAdminRequired
     * @param int $objectId
     * @param $dimension1
     * @param $dimension2
     * @param $value
     * @param bool $isDataset
     * @return array|false
     * @throws \OCP\DB\Exception
     */
    public function updateData(int $objectId, $dimension1, $dimension2, $value, bool $isDataset)
    {
        $dataset = $this->getDatasetId($objectId, $isDataset);

        if ($dataset != '') {
            $insert = $update = $errorMessage = 0;
            $action = array();
            $value = $this->floatvalue($value);
            if ($value === false) {
                $errorMessage = $this->l10n->t('3rd field must be a valid number');
            } else {
                $action = $this->StorageService->update($dataset, $dimension1, $dimension2, $value);
                $insert = $insert + $action['insert'];
                $update = $update + $action['update'];
            }

            $result = [
                'insert' => $insert,
                'update' => $update,
                'error' => $errorMessage,
                'validate' => $action['validate'],
            ];

            if ($errorMessage === 0) $this->ActivityManager->triggerEvent($dataset, ActivityManager::OBJECT_DATA, ActivityManager::SUBJECT_DATA_ADD);
            return $result;
        } else {
            return false;
        }
    }

    /**
     * Simulate delete data from input form
     *
     * @NoAdminRequired
     * @param int $objectId
     * @param $dimension1
     * @param $dimension2
     * @param bool $isDataset
     * @return array|false
     * @throws \OCP\DB\Exception
     */
    public function deleteDataSimulate(int $objectId, $dimension1, $dimension2, bool $isDataset)
    {
        $dataset = $this->getDatasetId($objectId, $isDataset);
        if ($dataset != '') {
            $result = $this->StorageService->deleteSimulate($dataset, $dimension1, $dimension2);
            return ['delete' => $result];
        } else {
            return false;
        }
    }

    /**
     * delete data from input form
     *
     * @NoAdminRequired
     * @param int $objectId
     * @param $dimension1
     * @param $dimension2
     * @param bool $isDataset
     * @return array|false
     * @throws \OCP\DB\Exception
     */
    public function deleteData(int $objectId, $dimension1, $dimension2, bool $isDataset)
    {
        $dataset = $this->getDatasetId($objectId, $isDataset);
        if ($dataset != '') {
            $result = $this->StorageService->delete($dataset, $dimension1, $dimension2);
            return ['delete' => $result];
        } else {
            return false;
        }
    }

    /**
     * Import clipboard data
     *
     * @NoAdminRequired
     * @param int $objectId
     * @param $import
     * @param bool $isDataset
     * @return array|false
     * @throws \OCP\DB\Exception
     */
    public function importClipboard($objectId, $import, bool $isDataset)
    {
        $dataset = $this->getDatasetId($objectId, $isDataset);
        if ($dataset != '') {
            $insert = $update = $errorMessage = $errorCounter = 0;
            $delimiter = '';

            if ($import === '') {
                $errorMessage = $this->l10n->t('No data');
            } else {
                $delimiter = $this->detectDelimiter($import);
                $rows = str_getcsv($import, "\n");

                foreach ($rows as &$row) {
                    $row = str_getcsv($row, $delimiter);
                    $numberOfColumns = count($row);
                    // last column needs to be a float
                    $row[2] = $this->floatvalue($row[$numberOfColumns - 1]);
                    if ($row[2] === false) {
                        $errorCounter++;
                    } else {
                        if ($numberOfColumns < 3) $row[1] = null;
                        $action = $this->StorageService->update($dataset, $row[0], $row[1], $row[2]);
                        $insert = $insert + $action['insert'];
                        $update = $update + $action['update'];
                    }
                    if ($errorCounter === 2) {
                        // first error is ignored; might be due to header row
                        $errorMessage = $this->l10n->t('Last field must be a valid number');
                        break;
                    }
                }
            }

            $result = [
                'insert' => $insert,
                'update' => $update,
                'delimiter' => $delimiter,
                'error' => $errorMessage,
            ];

            if ($errorMessage === 0) $this->ActivityManager->triggerEvent($dataset, ActivityManager::OBJECT_DATA, ActivityManager::SUBJECT_DATA_ADD_IMPORT);
            return $result;
        } else {
            return false;
        }
    }

    /**
     * Import data into dataset from an internal or external file
     *
     * @NoAdminRequired
     * @param int $objectId
     * @param $path
     * @param bool $isDataset
     * @return array|false
     * @throws \OCP\DB\Exception
     */
    public function importFile(int $objectId, $path, bool $isDataset)
    {
        $dataset = $this->getDatasetId($objectId, $isDataset);
        if ($dataset != '') {
            $insert = $update = 0;
            $reportMetadata['link'] = $path;

            $reportMetadata['user_id'] = $this->userId;
            $result = $this->DatasourceController->read(DatasourceController::DATASET_TYPE_FILE, $reportMetadata);

            if ($result['error'] === 0) {
                foreach ($result['data'] as &$row) {
                    $action = $this->StorageService->update($dataset, $row[0], $row[1], $row[2]);
                    $insert = $insert + $action['insert'];
                    $update = $update + $action['update'];
                }
            }

            $result = [
                'insert' => $insert,
                'update' => $update,
                'error' => $result['error'],
            ];

            if ($result['error'] === 0) $this->ActivityManager->triggerEvent($dataset, ActivityManager::OBJECT_DATA, ActivityManager::SUBJECT_DATA_ADD_IMPORT);
            return $result;
        } else {
            return false;
        }
    }

    private function getDatasetId($objectId, bool $isDataset)
    {
        if ($isDataset) {
            $dataset = $objectId;
        } else {
            $reportMetadata = $this->ReportService->read($objectId);
            $dataset = (int)$reportMetadata['dataset'];
        }
        return $dataset;
    }

    private function detectDelimiter($data): string
    {
        $delimiters = ["\t", ";", "|", ","];
        $data_2 = null;
        $delimiter = $delimiters[0];
        foreach ($delimiters as $d) {
            $firstRow = str_getcsv($data, "\n")[0];
            $data_1 = str_getcsv($firstRow, $d);
            if (sizeof($data_1) > sizeof($data_2)) {
                $delimiter = $d;
                $data_2 = $data_1;
            }
        }
        return $delimiter;
    }

    private function floatvalue($val)
    {
        $val = str_replace(",", ".", $val);
        $val = preg_replace('/\.(?=.*\.)/', '', $val);
        $val = preg_replace('/[^0-9-.]+/', '', $val);
        if (is_numeric($val)) {
            return number_format(floatval($val), 2, '.', '');
        } else {
            return false;
        }
    }

}

1			<?php
2			/**
3			* Analytics
4			*
5			* This file is licensed under the Affero General Public License version 3 or
6			* later. See the LICENSE.md file.
7			*
8			* @author Marcel Scherello <[email protected]>
9			* @copyright 2019-2022 Marcel Scherello
10			*/
11
12			namespace OCA\Analytics\Service;
13
14			use Exception;
15			use OCA\Analytics\Activity\ActivityManager;
16			use OCA\Analytics\Controller\DatasourceController;
17			use OCA\Analytics\Notification\NotificationManager;
18			use OCA\Analytics\Db\DataloadMapper;
19			use OCP\AppFramework\Http\NotFoundResponse;
20			use OCP\Files\NotFoundException;
21			use OCP\IL10N;
22			use Psr\Log\LoggerInterface;
23
24			class DataloadService
25			{
26			private $userId;
27			private $logger;
28			private $StorageService;
29			private $DatasourceController;
30			private $ActivityManager;
31			private $ReportService;
32			private $DatasetService;
33			private $VariableService;
34			private $l10n;
35			private $DataloadMapper;
36			private $NotificationManager;
37
38			public function __construct(
39			$userId,
40			IL10N $l10n,
41			LoggerInterface $logger,
42			ActivityManager $ActivityManager,
43			DatasourceController $DatasourceController,
44			ReportService $ReportService,
45			DatasetService $DatasetService,
46			StorageService $StorageService,
47			VariableService $VariableService,
48			NotificationManager $NotificationManager,
49			DataloadMapper $DataloadMapper
50			)
51			{
52			$this->userId = $userId;
53			$this->l10n = $l10n;
54			$this->logger = $logger;
55			$this->StorageService = $StorageService;
56			$this->ActivityManager = $ActivityManager;
57			$this->DatasourceController = $DatasourceController;
58			$this->ReportService = $ReportService;
59			$this->DatasetService = $DatasetService;
60			$this->VariableService = $VariableService;
61			$this->NotificationManager = $NotificationManager;
62			$this->DataloadMapper = $DataloadMapper;
63			}
64
65			// Data loads
66			// Data loads
67			// Data loads
68
69			/**
70			* create a new data load
71			*
72			* @param $datasetId
73			* @param $reportId
74			* @param int $datasourceId
75			* @return int
76			* @throws \OCP\DB\Exception
77			*/
78			public function create($datasetId, int $datasourceId)
79			{
80			return $this->DataloadMapper->create((int)$datasetId, $datasourceId);
81			}
82
83			/**
84			* get all data loads for a dataset or report
85			*
86			* @param int $datasetId
87			* @param $reportId
88			* @return array
89			*/
90			public function read($datasetId)
91			{
92			return $this->DataloadMapper->read((int)$datasetId);
93			}
94
95			/**
96			* update data load
97			*
98			* @param int $dataloadId
99			* @param $name
100			* @param $option
101			* @param $schedule
102			* @return bool
103			*/
104			public function update(int $dataloadId, $name, $option, $schedule)
105			{
106			$array = json_decode($option, true);
107			foreach ($array as $key => $value) {
108			$array[$key] = htmlspecialchars($value, ENT_NOQUOTES, 'UTF-8');
109			}
110			$option = json_encode($array);
111
112			return $this->DataloadMapper->update($dataloadId, $name, $option, $schedule);
113			}
114
115			/**
116			* delete a data load
117			*
118			* @param int $dataloadId
119			* @return bool
120			*/
121			public function delete(int $dataloadId)
122			{
123			return $this->DataloadMapper->delete($dataloadId);
124			}
125
126			/**
127			* execute all data loads depending on their schedule
128			* Daily or Hourly
129			*
130			* @param $schedule
131			* @return void
132			* @throws Exception
133			*/
134			public function executeBySchedule($schedule)
135			{
136			$schedules = $this->DataloadMapper->getDataloadBySchedule($schedule);
137			foreach ($schedules as $dataload) {
138			$result = $this->execute($dataload['id']);
139			if ($result['error'] !== 0) {
140			// if the data source produced an error, a notification needs to be triggered
141			$dataset = $this->DatasetService->read($dataload['dataset']);
142			$this->NotificationManager->triggerNotification(NotificationManager::DATALOAD_ERROR, $dataload['dataset'], $dataload['id'], ['dataloadName' => $dataload['name'], 'datasetName' => $dataset['name']], $dataload['user_id']);
143			}
144			}
145			}
146
147			/**
148			* execute a data load from data source and store into dataset
149			*
150			* @param int $dataloadId
151			* @return array
152			* @throws Exception
153			*/
154			public function execute(int $dataloadId)
155			{
156			$bulkSize = 500;
157			$insert = $update = $error = $delete = 0;
158			$bulkInsert = null;
159			$aggregation = null;
160
161			// get the meta data
162			$dataloadMetadata = $this->DataloadMapper->getDataloadById($dataloadId);
163			$option = json_decode($dataloadMetadata['option'], true);
164
165			// get the data from the datasource
166			$result = $this->getDataFromDatasource($dataloadId);
167			$datasetId = $result['datasetId'];
168
169			// dont continue in case of datasource error
170			if ($result['error'] !== 0) {
171			return [
172			'insert' => $insert,
173			'update' => $update,
174			'delete' => $delete,
175			'error' => 1,
176			];
177			}
178
179			// this is a deletion request. Just run the deletion and stop after that with a return.
180			if ($dataloadMetadata['datasource'] === 0) {
181			$option = json_decode($dataloadMetadata['option'], true);
182
183			// deletion jobs are using the same dimension/option/value settings a report filter
184			$filter['filteroptions'] = '{"filter":{"' . $option['filterDimension'] . '":{"option":"' . $option['filterOption'] . '","value":"' . $option['filterValue'] . '"}}}';
			0 ignored issues – show Comprehensibility Best Practice introduced 2022-12-18 21:37 UTC by Report Bug Copy Issue Report `$filter` was never initialized. Although not strictly required by PHP, it is generally a good practice to add `$filter = array();` before regardless. Loading history...
185			// Text variables like %xx% could be in use here
186			$filter = $this->VariableService->replaceFilterVariables($filter);
187
188			$this->StorageService->deleteWithFilterSimulate($dataloadMetadata['dataset'], json_decode($filter['filteroptions'], true));
189
190			return [
191			'insert' => $insert,
192			'update' => $update,
193			// use the existing row count from the deletion simulation which was run during getDataFromDatasource
194			'delete' => $result['data']['count'],
195			'error' => $error,
196			];
197			}
198
199			// "delete all date before loading" is true in the data source options
200			// in this case, bulkInsert is set to true. Then no further checks for existing records are needed
201			// reduce db selects
202			if (isset($option['delete']) and $option['delete'] === 'true') {
203			$this->StorageService->delete($datasetId, '', '', $dataloadMetadata['user_id']);
204			$bulkInsert = true;
205			}
206
207			// if the data set has no data, it is the same as the delete all option
208			// in this case, bulkInsert is set to true. Then no further checks for existing records are needed
209			// reduce db selects
210			$numberOfRecords = $this->StorageService->getRecordCount($datasetId);
211			if ($numberOfRecords['count'] === 0) {
212			$this->logger->info('Empty dataset => enabling bulk load');
213			$bulkInsert = true;
214			}
215
216			// Feature not yet available
217			if (isset($option['aggregation']) and $option['aggregation'] !== 'overwrite') {
218			$aggregation = $option['aggregation'];
219			}
220
221			// collect mass updates to reduce statements to the database
222			$this->DataloadMapper->beginTransaction();
223			$currentCount = 0;
224			foreach ($result['data'] as $row) {
225			if (count($row) === 1) {
226			// only one column is not possible
227			$this->logger->info('loading data with only one column is not possible. This is a data load for the dataset: ' . $datasetId);
228			$error = $error + 1;
229			continue;
230			}
231			if (count($row) === 2) {
232			// if data source only delivers 2 columns, the value needs to be in the last one
233			$row[2] = $row[1];
234			$row[1] = null;
235			}
236			$action = $this->StorageService->update($datasetId, $row[0], $row[1], $row[2], $dataloadMetadata['user_id'], $bulkInsert, $aggregation);
237			$insert = $insert + $action['insert'];
238			$update = $update + $action['update'];
239			$error = $error + $action['error'];
240
241			if ($currentCount % $bulkSize === 0) {
242			$this->DataloadMapper->commit();
243			$this->DataloadMapper->beginTransaction();
244			}
245			if ($action['error'] === 0) $currentCount++;
246			}
247			$this->DataloadMapper->commit();
248
249			$result = [
250			'insert' => $insert,
251			'update' => $update,
252			'delete' => $delete,
253			'error' => $error,
254			];
255
256			$this->ActivityManager->triggerEvent($datasetId, ActivityManager::OBJECT_DATA, ActivityManager::SUBJECT_DATA_ADD_DATALOAD, $dataloadMetadata['user_id']);
257			return $result;
258			}
259
260			/**
261			* get the data from datasource
262			* to be used in simulation or execution
263			*
264			* @param int $dataloadId
265			* @return array\|NotFoundResponse
266			* @throws NotFoundResponse\|NotFoundException
267			* @throws \OCP\DB\Exception
268			*/
269			public function getDataFromDatasource(int $dataloadId)
270			{
271			$dataloadMetadata = $this->DataloadMapper->getDataloadById($dataloadId);
272
273			if (!empty($dataloadMetadata)) {
274
275			if ($dataloadMetadata['datasource'] !== 0) {
276			$dataloadMetadata['link'] = $dataloadMetadata['option']; //remap until data source table is renamed link=>option
277			$result = $this->DatasourceController->read((int)$dataloadMetadata['datasource'], $dataloadMetadata);
278			$result['datasetId'] = $dataloadMetadata['dataset'];
279			} else {
280			// this is a deletion request. Just run the simulation and return the possible row count in the expected result array
281			$option = json_decode($dataloadMetadata['option'], true);
282
283			// deletion jobs are using the same dimension/option/value settings a report filter
284			$filter['filteroptions'] = '{"filter":{"' . $option['filterDimension'] . '":{"option":"' . $option['filterOption'] . '","value":"' . $option['filterValue'] . '"}}}';
			0 ignored issues – show Comprehensibility Best Practice introduced 2022-12-18 21:37 UTC by Report Bug Copy Issue Report `$filter` was never initialized. Although not strictly required by PHP, it is generally a good practice to add `$filter = array();` before regardless. Loading history...
285			// Text variables like %xx% could be in use here
286			$filter = $this->VariableService->replaceFilterVariables($filter);
287
288			$result = [
289			'header' => '',
290			'dimensions' => '',
291			'data' => $this->StorageService->deleteWithFilterSimulate($dataloadMetadata['dataset'], json_decode($filter['filteroptions'], true)),
292			'error' => 0,
293			];
294			}
295
296			return $result;
			0 ignored issues – show Bug Best Practice introduced 2021-02-08 19:35 UTC by Report Bug Copy Issue Report The expression `return $result` also could return the type `OCP\Files\NotFoundException` which is incompatible with the documented return type `OCP\AppFramework\Http\NotFoundResponse\|array`. Loading history...
297			} else {
298			return new NotFoundResponse();
299			}
300			}
301
302			// Data Manipulation
303			// Data Manipulation
304			// Data Manipulation
305
306			/**
307			* update data from input form
308			*
309			* @NoAdminRequired
310			* @param int $objectId
311			* @param $dimension1
312			* @param $dimension2
313			* @param $value
314			* @param bool $isDataset
315			* @return array\|false
316			* @throws \OCP\DB\Exception
317			*/
318			public function updateData(int $objectId, $dimension1, $dimension2, $value, bool $isDataset)
319			{
320			$dataset = $this->getDatasetId($objectId, $isDataset);
321
322			if ($dataset != '') {
323			$insert = $update = $errorMessage = 0;
324			$action = array();
325			$value = $this->floatvalue($value);
326			if ($value === false) {
327			$errorMessage = $this->l10n->t('3rd field must be a valid number');
328			} else {
329			$action = $this->StorageService->update($dataset, $dimension1, $dimension2, $value);
330			$insert = $insert + $action['insert'];
331			$update = $update + $action['update'];
332			}
333
334			$result = [
335			'insert' => $insert,
336			'update' => $update,
337			'error' => $errorMessage,
338			'validate' => $action['validate'],
339			];
340
341			if ($errorMessage === 0) $this->ActivityManager->triggerEvent($dataset, ActivityManager::OBJECT_DATA, ActivityManager::SUBJECT_DATA_ADD);
342			return $result;
343			} else {
344			return false;
345			}
346			}
347
348			/**
349			* Simulate delete data from input form
350			*
351			* @NoAdminRequired
352			* @param int $objectId
353			* @param $dimension1
354			* @param $dimension2
355			* @param bool $isDataset
356			* @return array\|false
357			* @throws \OCP\DB\Exception
358			*/
359			public function deleteDataSimulate(int $objectId, $dimension1, $dimension2, bool $isDataset)
360			{
361			$dataset = $this->getDatasetId($objectId, $isDataset);
362			if ($dataset != '') {
363			$result = $this->StorageService->deleteSimulate($dataset, $dimension1, $dimension2);
364			return ['delete' => $result];
365			} else {
366			return false;
367			}
368			}
369
370			/**
371			* delete data from input form
372			*
373			* @NoAdminRequired
374			* @param int $objectId
375			* @param $dimension1
376			* @param $dimension2
377			* @param bool $isDataset
378			* @return array\|false
379			* @throws \OCP\DB\Exception
380			*/
381			public function deleteData(int $objectId, $dimension1, $dimension2, bool $isDataset)
382			{
383			$dataset = $this->getDatasetId($objectId, $isDataset);
384			if ($dataset != '') {
385			$result = $this->StorageService->delete($dataset, $dimension1, $dimension2);
386			return ['delete' => $result];
387			} else {
388			return false;
389			}
390			}
391
392			/**
393			* Import clipboard data
394			*
395			* @NoAdminRequired
396			* @param int $objectId
397			* @param $import
398			* @param bool $isDataset
399			* @return array\|false
400			* @throws \OCP\DB\Exception
401			*/
402			public function importClipboard($objectId, $import, bool $isDataset)
403			{
404			$dataset = $this->getDatasetId($objectId, $isDataset);
405			if ($dataset != '') {
406			$insert = $update = $errorMessage = $errorCounter = 0;
407			$delimiter = '';
408
409			if ($import === '') {
410			$errorMessage = $this->l10n->t('No data');
411			} else {
412			$delimiter = $this->detectDelimiter($import);
413			$rows = str_getcsv($import, "\n");
414
415			foreach ($rows as &$row) {
416			$row = str_getcsv($row, $delimiter);
417			$numberOfColumns = count($row);
418			// last column needs to be a float
419			$row[2] = $this->floatvalue($row[$numberOfColumns - 1]);
420			if ($row[2] === false) {
421			$errorCounter++;
422			} else {
423			if ($numberOfColumns < 3) $row[1] = null;
424			$action = $this->StorageService->update($dataset, $row[0], $row[1], $row[2]);
425			$insert = $insert + $action['insert'];
426			$update = $update + $action['update'];
427			}
428			if ($errorCounter === 2) {
429			// first error is ignored; might be due to header row
430			$errorMessage = $this->l10n->t('Last field must be a valid number');
431			break;
432			}
433			}
434			}
435
436			$result = [
437			'insert' => $insert,
438			'update' => $update,
439			'delimiter' => $delimiter,
440			'error' => $errorMessage,
441			];
442
443			if ($errorMessage === 0) $this->ActivityManager->triggerEvent($dataset, ActivityManager::OBJECT_DATA, ActivityManager::SUBJECT_DATA_ADD_IMPORT);
444			return $result;
445			} else {
446			return false;
447			}
448			}
449
450			/**
451			* Import data into dataset from an internal or external file
452			*
453			* @NoAdminRequired
454			* @param int $objectId
455			* @param $path
456			* @param bool $isDataset
457			* @return array\|false
458			* @throws \OCP\DB\Exception
459			*/
460			public function importFile(int $objectId, $path, bool $isDataset)
461			{
462			$dataset = $this->getDatasetId($objectId, $isDataset);
463			if ($dataset != '') {
464			$insert = $update = 0;
465			$reportMetadata['link'] = $path;
			0 ignored issues – show Comprehensibility Best Practice introduced 2021-11-20 21:09 UTC by Report Bug Copy Issue Report `$reportMetadata` was never initialized. Although not strictly required by PHP, it is generally a good practice to add `$reportMetadata = array();` before regardless. Loading history...
466			$reportMetadata['user_id'] = $this->userId;
467			$result = $this->DatasourceController->read(DatasourceController::DATASET_TYPE_FILE, $reportMetadata);
468
469			if ($result['error'] === 0) {
470			foreach ($result['data'] as &$row) {
471			$action = $this->StorageService->update($dataset, $row[0], $row[1], $row[2]);
472			$insert = $insert + $action['insert'];
473			$update = $update + $action['update'];
474			}
475			}
476
477			$result = [
478			'insert' => $insert,
479			'update' => $update,
480			'error' => $result['error'],
481			];
482
483			if ($result['error'] === 0) $this->ActivityManager->triggerEvent($dataset, ActivityManager::OBJECT_DATA, ActivityManager::SUBJECT_DATA_ADD_IMPORT);
484			return $result;
485			} else {
486			return false;
487			}
488			}
489
490			private function getDatasetId($objectId, bool $isDataset)
491			{
492			if ($isDataset) {
493			$dataset = $objectId;
494			} else {
495			$reportMetadata = $this->ReportService->read($objectId);
496			$dataset = (int)$reportMetadata['dataset'];
497			}
498			return $dataset;
499			}
500
501			private function detectDelimiter($data): string
502			{
503			$delimiters = ["\t", ";", "\|", ","];
504			$data_2 = null;
505			$delimiter = $delimiters[0];
506			foreach ($delimiters as $d) {
507			$firstRow = str_getcsv($data, "\n")[0];
508			$data_1 = str_getcsv($firstRow, $d);
509			if (sizeof($data_1) > sizeof($data_2)) {
510			$delimiter = $d;
511			$data_2 = $data_1;
512			}
513			}
514			return $delimiter;
515			}
516
517			private function floatvalue($val)
518			{
519			$val = str_replace(",", ".", $val);
520			$val = preg_replace('/\.(?=.*\.)/', '', $val);
521			$val = preg_replace('/[^0-9-.]+/', '', $val);
522			if (is_numeric($val)) {
523			return number_format(floatval($val), 2, '.', '');
524			} else {
525			return false;
526			}
527			}
528
529			}

Rello / analytics

Push — master ( 1876ce...d1e05a )

DataloadService::executeBySchedule() A

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like