1
|
|
|
<?php |
2
|
|
|
|
3
|
|
|
namespace PhpTek\Exodus\Transform; |
4
|
|
|
|
5
|
|
|
use SilverStripe\Assets\File; |
6
|
|
|
use SilverStripe\Assets\Folder; |
7
|
|
|
use SilverStripe\Control\Controller; |
8
|
|
|
use SilverStripe\ORM\DataObject; |
9
|
|
|
use SilverStripe\AssetAdmin\Helper\ImageThumbnailHelper; |
|
|
|
|
10
|
|
|
|
11
|
|
|
/** |
12
|
|
|
* URL transformer specific to SilverStripe's `File` class for use with the module's |
13
|
|
|
* import content feature. It will re-create all available data of the scraped file into SilverStripe's |
14
|
|
|
* database and re-create a copy of the file itself on the filesystem. |
15
|
|
|
* If enabled in the CMS UI, links to imported images and documents in imported page-content will also be automatically |
16
|
|
|
* re-written. |
17
|
|
|
* |
18
|
|
|
* @todo write unit-test for unwritable assets dir. |
19
|
|
|
* |
20
|
|
|
* @package phptek/silverstripe-exodus |
21
|
|
|
* @author Sam Minee <[email protected]> |
22
|
|
|
* @author Russell Michell <[email protected]> |
23
|
|
|
* @see {@link StaticSiteDataTypeTransformer} |
24
|
|
|
*/ |
25
|
|
|
class StaticSiteFileTransformer extends StaticSiteDataTypeTransformer |
26
|
|
|
{ |
27
|
|
|
/** |
28
|
|
|
* Default value to pass to usleep() to reduce load on the remote server |
29
|
|
|
* |
30
|
|
|
* @var number |
31
|
|
|
*/ |
32
|
|
|
private static $sleep_multiplier = 10; |
33
|
|
|
|
34
|
|
|
/** |
35
|
|
|
* Generic function called by \ExternalContentImporter |
36
|
|
|
* |
37
|
|
|
* @inheritdoc |
38
|
|
|
* |
39
|
|
|
* @param bool $genThumbs Used to prevent errors in thumbnail generation when running tests |
40
|
|
|
*/ |
41
|
|
|
public function transform($item, $parentObject, $strategy, $genThumbs = true) |
42
|
|
|
{ |
43
|
|
|
$this->utils->log("START file-transform for: ", $item->AbsoluteURL, $item->ProcessedMIME); |
44
|
|
|
|
45
|
|
|
if (!$item->checkIsType('file')) { |
46
|
|
|
$this->utils->log(" - Item not of type \'file\'. for: ", $item->AbsoluteURL, $item->ProcessedMIME); |
47
|
|
|
$this->utils->log("END page-transform for: ", $item->AbsoluteURL, $item->ProcessedMIME); |
48
|
|
|
|
49
|
|
|
return false; |
|
|
|
|
50
|
|
|
} |
51
|
|
|
|
52
|
|
|
$source = $item->getSource(); |
53
|
|
|
|
54
|
|
|
// Sleep for Xms to reduce load on the remote server |
55
|
|
|
usleep((int) self::$sleep_multiplier * 1000); |
56
|
|
|
|
57
|
|
|
// Extract remote location of File |
58
|
|
|
$contentFields = $this->getContentFieldsAndSelectors($item, 'File'); |
59
|
|
|
|
60
|
|
|
// Default value for Title |
61
|
|
|
if (empty($contentFields['Filename'])) { |
62
|
|
|
$contentFields['Filename'] = ['content' => $item->externalId]; |
63
|
|
|
} |
64
|
|
|
|
65
|
|
|
$schema = $source->getSchemaForURL($item->AbsoluteURL, $item->ProcessedMIME); |
66
|
|
|
|
67
|
|
|
if (!$schema) { |
68
|
|
|
$this->utils->log(" - Couldn't find an import schema for: ", $item->AbsoluteURL, $item->ProcessedMIME); |
69
|
|
|
$this->utils->log("END file-transform for: ", $item->AbsoluteURL, $item->ProcessedMIME); |
70
|
|
|
return false; |
|
|
|
|
71
|
|
|
} |
72
|
|
|
|
73
|
|
|
$dataType = $schema->DataType; |
74
|
|
|
|
75
|
|
|
if (!$dataType) { |
76
|
|
|
$this->utils->log(" - DataType for migration schema is empty for: ", $item->AbsoluteURL, $item->ProcessedMIME); |
77
|
|
|
$this->utils->log("END file-transform for: ", $item->AbsoluteURL, $item->ProcessedMIME); |
78
|
|
|
throw new \Exception('DataType for migration schema is empty!'); |
79
|
|
|
} |
80
|
|
|
|
81
|
|
|
// Process incoming according to user-selected duplication strategy |
82
|
|
|
if (!$file = $this->duplicationStrategy($dataType, $item, $source->BaseUrl, $strategy, $parentObject)) { |
83
|
|
|
$this->utils->log("END file-transform for: ", $item->AbsoluteURL, $item->ProcessedMIME); |
84
|
|
|
return false; |
|
|
|
|
85
|
|
|
} |
86
|
|
|
|
87
|
|
|
// Prepare $file with all the correct properties, ready for writing |
88
|
|
|
$tmpPath = $contentFields['tmp_path']; |
89
|
|
|
|
90
|
|
|
if (!$file = $this->buildFileProperties($file, $item, $source, $tmpPath)) { |
|
|
|
|
91
|
|
|
$this->utils->log("END file-transform for: ", $item->AbsoluteURL, $item->ProcessedMIME); |
92
|
|
|
return false; |
|
|
|
|
93
|
|
|
} |
94
|
|
|
|
95
|
|
|
try { |
96
|
|
|
if (!$file->write()) { |
97
|
|
|
$this->utils->log(" - Not imported (no write): ", $item->AbsoluteURL, $item->ProcessedMIME); |
98
|
|
|
} |
99
|
|
|
|
100
|
|
|
// Remove garbage tmp files if/when left lying around |
101
|
|
|
if (file_exists($tmpPath)) { |
102
|
|
|
unlink($tmpPath); |
103
|
|
|
} |
104
|
|
|
|
105
|
|
|
$file->publishSingle(); |
106
|
|
|
|
107
|
|
|
// Generate thumbnails |
108
|
|
|
if ($genThumbs) { |
109
|
|
|
ImageThumbnailHelper::singleton()->run(); |
110
|
|
|
} |
111
|
|
|
} catch (\Exception $e) { |
112
|
|
|
$this->utils->log($e->getMessage(), $item->AbsoluteURL, $item->ProcessedMIME); |
113
|
|
|
} |
114
|
|
|
|
115
|
|
|
$this->utils->log("END file-transform for: ", $item->AbsoluteURL, $item->ProcessedMIME); |
116
|
|
|
|
117
|
|
|
return StaticSiteTransformResult::create($file, $item->stageChildren()); |
118
|
|
|
} |
119
|
|
|
|
120
|
|
|
/** |
121
|
|
|
* Build the properties required for a safely saved SilverStripe asset. |
122
|
|
|
* Attempts to detect and fix bad file-extensions based on the available Mime-Type. |
123
|
|
|
* |
124
|
|
|
* @param File $file |
125
|
|
|
* @param StaticSiteContentItem $item Object properties are used to fixup bad-file extensions or filenames with no |
|
|
|
|
126
|
|
|
* extension but which _do_ have a Mime-Type. |
127
|
|
|
* @param StaticSiteContentSource $source Content source object |
|
|
|
|
128
|
|
|
* @param string $tmpPath Path to partially uploaded file object or null (unit-tests) |
129
|
|
|
* @return mixed (boolean | File) |
130
|
|
|
*/ |
131
|
|
|
public function buildFileProperties(File $file, $item, $source, $tmpPath = null) |
132
|
|
|
{ |
133
|
|
|
$url = $item->AbsoluteURL; |
134
|
|
|
$mime = $item->ProcessedMIME; |
135
|
|
|
$assetsPath = $this->getDirHierarchy($url); |
136
|
|
|
|
137
|
|
|
/* |
138
|
|
|
* Run checks on original filename and name it as per default if nothing can be done with it. |
139
|
|
|
* '.zzz' not in framework/_config/mimetypes.yml and unlikely ever to be found in File, so fails gracefully. |
140
|
|
|
*/ |
141
|
|
|
$dummy = 'unknown.zzz'; |
142
|
|
|
$origFilename = pathinfo($url, PATHINFO_FILENAME); |
143
|
|
|
$origFilename = (mb_strlen($origFilename) > 0 ? $origFilename : $dummy); |
|
|
|
|
144
|
|
|
|
145
|
|
|
/* |
146
|
|
|
* Some assets come through with no file-extension, which confuses SS's File logic |
147
|
|
|
* and throws errors causing the import to stop dead. |
148
|
|
|
* Check for this and guess an appropriate file-extension, if possible. |
149
|
|
|
*/ |
150
|
|
|
$oldExt = pathinfo($url, PATHINFO_EXTENSION); |
151
|
|
|
$extIsValid = in_array($oldExt, $this->getSSExtensions()); |
152
|
|
|
// Only attempt to define and append a new filename ($newExt) if $oldExt is invalid |
153
|
|
|
$newExt = null; |
154
|
|
|
|
155
|
|
|
if (!$extIsValid && !$newExt = $this->mimeProcessor->ext_to_mime_compare($oldExt, $mime, true)) { |
|
|
|
|
156
|
|
|
$this->utils->log(" - WARNING: Bad file-extension: \"$oldExt\". Unable to assign new file-extension (#1) - DISCARDING.", $url, $mime); |
157
|
|
|
|
158
|
|
|
return false; |
159
|
|
|
} elseif ($newExt) { |
160
|
|
|
$useExtension = $newExt; |
161
|
|
|
$logMessagePt1 = "NOTICE: Bad file-extension: \"$oldExt\". Assigned new file-extension: \"$newExt\" based on MimeType."; |
162
|
|
|
$logMessagePt2 = PHP_EOL . "\t - FROM: \"$url\"" . PHP_EOL . "\t - TO: \"$origFilename.$newExt\""; |
163
|
|
|
|
164
|
|
|
$this->utils->log(' - ' . $logMessagePt1 . $logMessagePt2, '', $mime); |
165
|
|
|
} else { |
166
|
|
|
// If $newExt didn't work, check again if $oldExt is invalid and just lose it. |
167
|
|
|
if (!$extIsValid) { |
168
|
|
|
$this->utils->log(" - WARNING: Bad file-extension: \"$oldExt\". Unable to assign new file-extension (#2) - DISCARDING.", $url, $mime); |
169
|
|
|
|
170
|
|
|
return false; |
171
|
|
|
} |
172
|
|
|
|
173
|
|
|
if ($this->mimeProcessor->isBadMimeType($mime)) { |
174
|
|
|
$this->utils->log(" - WARNING: Bad mime-type: \"$mime\". Unable to assign new file-extension (#3) - DISCARDING.", $url, $mime); |
175
|
|
|
|
176
|
|
|
return false; |
177
|
|
|
} |
178
|
|
|
|
179
|
|
|
$useExtension = $oldExt; |
180
|
|
|
} |
181
|
|
|
|
182
|
|
|
$folder = Folder::find_or_make($assetsPath); |
183
|
|
|
$fileName = sprintf('%s.%s', $origFilename, $useExtension); |
|
|
|
|
184
|
|
|
|
185
|
|
|
// Only ever passed as null from tests |
186
|
|
|
if ($tmpPath) { |
187
|
|
|
$file->setFromLocalFile($tmpPath, $fileName); |
188
|
|
|
} |
189
|
|
|
|
190
|
|
|
$file->setFilename($fileName); |
191
|
|
|
$file->ParentID = $folder->ID; |
192
|
|
|
$file->StaticSiteContentSourceID = $source->ID; |
193
|
|
|
$file->StaticSiteURL = $url; |
194
|
|
|
$file->StaticSiteImportID = $this->getCurrentImportID(); |
195
|
|
|
|
196
|
|
|
$this->utils->log(" - NOTICE: \"File-properties built successfully for: ", $url, $mime); |
197
|
|
|
|
198
|
|
|
return $file; |
199
|
|
|
} |
200
|
|
|
|
201
|
|
|
/** |
202
|
|
|
* Determine the correct parent directory hierarchy from the imported file's remote-path, |
203
|
|
|
* such that it is mapped to the appropriate area under the main SilverStripe 'assets' directory. |
204
|
|
|
* |
205
|
|
|
* @param string $absolutePath The absolute path of this file on the remote server. |
206
|
|
|
* @param boolean $full Return absolute path from server's filesystem root |
207
|
|
|
* @return string The path to append to 'assets' and use as local cache dir. |
208
|
|
|
*/ |
209
|
|
|
public function getDirHierarchy(string $absoluteUrl, bool $full = false): string |
210
|
|
|
{ |
211
|
|
|
/* |
212
|
|
|
* Determine the top-level directory under 'assets' under-which this item's |
213
|
|
|
* dir-hierarchy will be created. |
214
|
|
|
*/ |
215
|
|
|
$parentDir = ''; |
216
|
|
|
$postVars = Controller::curr()->request->postVars(); |
217
|
|
|
|
218
|
|
|
if (!empty($postVars['FileMigrationTarget'])) { |
219
|
|
|
$parentDirData = DataObject::get_by_id(File::class, $postVars['FileMigrationTarget']); |
220
|
|
|
$parentDir = $parentDirData->Title; |
221
|
|
|
} |
222
|
|
|
|
223
|
|
|
$replaceUnused = preg_replace("#https?://(www.)?[^/]+#", '', $absoluteUrl); |
224
|
|
|
$fragments = explode('/', $replaceUnused); |
225
|
|
|
$filename = pathinfo($absoluteUrl, PATHINFO_FILENAME); |
226
|
|
|
$path = []; |
227
|
|
|
|
228
|
|
|
foreach ($fragments as $fragment) { |
229
|
|
|
$dontUse = (!strlen($fragment) || preg_match("#(http|$filename|www\.)+#", $fragment)); |
230
|
|
|
|
231
|
|
|
if ($dontUse) { |
232
|
|
|
continue; |
233
|
|
|
} |
234
|
|
|
|
235
|
|
|
array_push($path, $fragment); |
236
|
|
|
} |
237
|
|
|
|
238
|
|
|
$joinedPath = Controller::join_links($parentDir, implode('/', $path)); |
239
|
|
|
$fullPath = ASSETS_PATH . ($joinedPath ? DIRECTORY_SEPARATOR . $joinedPath : ''); |
240
|
|
|
|
241
|
|
|
return $full ? $fullPath : $joinedPath; |
242
|
|
|
} |
243
|
|
|
} |
244
|
|
|
|
The issue could also be caused by a filter entry in the build configuration. If the path has been excluded in your configuration, e.g.
excluded_paths: ["lib/*"]
, you can move it to the dependency path list as follows:For further information see https://scrutinizer-ci.com/docs/tools/php/php-scrutinizer/#list-dependency-paths