FactsFactory::REMOVE_EMPTY() - Code Metrics - Inspection of "Dev8.2" - linkeddatacenter/BOTK-core - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Pull Request — master (#14)

by Enrico

created 2020-12-05 22:30 UTC

FactsFactory::REMOVE_EMPTY() A

↳ Parent: FactsFactory

Complexity

Conditions	3
Paths	3

Size

Total Lines

Duplication

Lines	0
Ratio	0 %

Code Coverage

Tests	5
CRAP Score	3

Importance

Changes

Metric	Value
dl	0
loc	8
ccs	5
cts	5
cp	1
rs	10
c	0
b	0
f	0
cc	3
nc	3
nop	1
crap	3

<?php
namespace BOTK;

use BOTK\Exception\FactsValidatorWarning;
use BOTK\Exception\InvalidRawDataWarning;
use BOTK\Exception\TooManyErrorsException;
use BOTK\Exception\TooManyInsanesException;

/**
 * Create structured data from an array of raw data (i.e. ie just a sequence of scalars) managing errors.
 * The class provides a RDF triple counter and tresholds for data processinge errors
 * Options:
 * 	'datamapper' a callable that accepts a raw data array and create structured data as an array. Must be provided.
 *  'rawdataSanitizer a callable that validate raw data before datamapper. It returns an array of raw data of false if rawdata is invalid.
 *  'dataCleaner' a callable that filters the structured data returned by datamapper before to instanciate data model, 
 * 					by default it removes all empty properties.
 *  'factsErrorDetector' a callable that validate computed facts. It accepts a ModelInterface and returns an error description  or false. 
 * 					By defaults accepted raw data that produces empty facts are considered errors.
 */
class FactsFactory implements FactsFactoryInterface {
	
	protected $profile;
	protected $modelClass;
	protected $counter = array(
		'triple'		=> 0,			// rdf triples in facts
		'error'			=> 0,			// facts contains error
		'insane'		=> 0,			// raw data unaccepted
		'entity'		=> 0,			// raw data processed
	);
	
	
	public function __construct( array $profile =array() )
	{
		$defaults = array(
			'model'					  => 'SampleSchemaThing',
			'modelOptions'			  => array(),
			'entityThreshold'		  => 100, // min numbers of entity that trigger error resilence computation.
			'resilienceToErrors' 	  => 0.3, // if more than 30% of error throws a TooManyErrorException
			'resilienceToInsanes'	  => 0.9, // if more than 90% of unacceptable data throws a TooManyErrorException
			'source' 			  	  => null,	
			'datamapper'			  => function($rawdata){return $rawdata;},
			'dataCleaner' 		  	  => get_class().'::REMOVE_EMPTY',
			'factsErrorDetector' 	  => get_class().'::NOT_EMPTY_FACTS',
			'rawdataSanitizer' 		  => function($rawdata){return is_array($rawdata)?$rawdata:false;},
		);
		$this->profile = array_merge($defaults,$profile);
		$this->modelClass = class_exists($this->profile['model'])
			?$this->profile['model']
			:('\BOTK\Model\\'.$this->profile['model']);
		
		if( !class_exists($this->modelClass) || !is_subclass_of($this->modelClass, '\BOTK\ModelInterface')){

			throw new \InvalidArgumentException("The provided model ({$this->profile['model']} is unknown");	
		}
		if( !is_callable($this->profile['datamapper'])) {
			throw new \InvalidArgumentException("Invalid datamapper callback");	
		}
		if( !is_callable($this->profile['dataCleaner'])) {
			throw new \InvalidArgumentException("Invalid dataCleaner callback");	
		}
		if( !is_callable($this->profile['rawdataSanitizer'])) {
			throw new \InvalidArgumentException("Invalid rawdataSanitizer callback");	
		}
		if( !is_callable($this->profile['factsErrorDetector'])) {
			throw new \InvalidArgumentException("Invalid factsErrorDetector callback");	
		}
	}
	
	
	/**
	 * two level filter array, a default for dataCleaner callback
	 */
	public static function REMOVE_EMPTY( array $data)
	{
		$a = array();
	    foreach ($data as $key => $value) {
	       $a[$key] = is_array($value)?array_filter($value):$value;
	    }
	    return array_filter($a);
	}
	
	/**
	 * a default for dataValidator callback  
	 */
	public static function NOT_EMPTY_FACTS( \BOTK\ModelInterface $data)
	{
		return $data->getTripleCount()?false:'No facts found.';
	}
	
	
	/**
	 * create facts from rawdata. Please nothe that null facts does not means always an error (i.e. no facts is a fact).
	 * if you do not want empty facts use dataValidator
	 */
	public function factualize($rawData)
	{
		$rawdataSanitizer = $this->profile['rawdataSanitizer'];
		$validRawData = $rawdataSanitizer($rawData);
		$this->counter['entity']++;
		
		if (!empty($validRawData)){
			$datamapper = $this->profile['datamapper'];
			$dataCleaner = $this->profile['dataCleaner'];
			$factsErrorDetector = $this->profile['factsErrorDetector'];
			$data =$dataCleaner($datamapper($validRawData));
			$facts = call_user_func($this->modelClass.'::fromArray',$data,$this->profile['modelOptions']);
			$this->counter['triple'] += $facts->getTripleCount();
			if($error=$factsErrorDetector($facts)){
				$this->counter['error']++;
				throw new FactsValidatorWarning($error,$facts);
			}
		} else {
			$this->counter['insane']++;
			throw new InvalidRawDataWarning("Invalid rawdata",$rawData);
		}
		
		// ensure that not too many errors
		$errorRate = ($this->counter['error']/$this->counter['entity']);
		if(( $this->counter['entity'] > $this->profile['entityThreshold']) 
				&& ( $errorRate > $this->profile['resilienceToErrors'])){
			$x = $this->profile['resilienceToErrors']*100;
			throw new TooManyErrorsException("Error rate in data processing exceeded the $x% threshold");			
		}

		// ensure that not too many insaness raw data
		$insaneRate = ($this->counter['insane']/$this->counter['entity']);
		if(( $this->counter['entity'] > $this->profile['entityThreshold']) 
				&& ($insaneRate > $this->profile['resilienceToInsanes'])){
			$x = $this->profile['resilienceToInsanes']*100;
			throw new TooManyInsanesException("Unacceptable data rate exceeded the $x% threshold");			
		}

		return $facts;
	}
	
	
	public function generateLinkedDataHeader()
	{
		return call_user_func($this->modelClass.'::getTurtleHeader'); 
	}
	
	
	public function generateLinkedDataFooter()
	{
		$now = date('c');
		$rdf = "
#@prefix dct: <http://purl.org/dc/terms/> .
#@prefix void: <http://rdfs.org/ns/void#> .
#@prefix prov: <http://www.w3.org/ns/prov#> .
";

		// add  provenance info
		$rdf .= "#<> prov:generatedAtTime \"$now\"^^xsd:dateTime;";
		if(!empty($this->profile['source'])){
			$rdf.= "dct:source <{$this->profile['source']}>;";	
		}
		
		
		// add dataset info and a human readable comment as last line
		$rdf.= "foaf:primaryTopic <#dataset>.\n";
		$rdf.= "#<#dataset> a void:Dataset; void:datadump <>;void:triples {$this->counter['triple']} ;void:entities {$this->counter['entity']}.\n";
		$rdf.= "#Generated {$this->counter['triple']} good triples from {$this->counter['entity']} entities ({$this->counter['insane']} ignored), {$this->counter['error']} errors\n";
		
		return $rdf;
	}


	public function addToCounter($counter,$val=1)
	{
		if(!array_key_exists($counter,$this->counter)){
			throw new \InvalidArgumentException("Invalid counter name");
		}
		$this->counter[$counter]+= intval($val);
	}
	
	
	public function getCounters()
	{
		return $this->counter;
	}

}

1		<?php
2		namespace BOTK;
3
4		use BOTK\Exception\FactsValidatorWarning;
5		use BOTK\Exception\InvalidRawDataWarning;
6		use BOTK\Exception\TooManyErrorsException;
7		use BOTK\Exception\TooManyInsanesException;
8
9		/**
10		* Create structured data from an array of raw data (i.e. ie just a sequence of scalars) managing errors.
11		* The class provides a RDF triple counter and tresholds for data processinge errors
12		* Options:
13		* 'datamapper' a callable that accepts a raw data array and create structured data as an array. Must be provided.
14		* 'rawdataSanitizer a callable that validate raw data before datamapper. It returns an array of raw data of false if rawdata is invalid.
15		* 'dataCleaner' a callable that filters the structured data returned by datamapper before to instanciate data model,
16		* by default it removes all empty properties.
17		* 'factsErrorDetector' a callable that validate computed facts. It accepts a ModelInterface and returns an error description or false.
18		* By defaults accepted raw data that produces empty facts are considered errors.
19		*/
20		class FactsFactory implements FactsFactoryInterface {
21
22		protected $profile;
23		protected $modelClass;
24		protected $counter = array(
25		'triple' => 0, // rdf triples in facts
26		'error' => 0, // facts contains error
27		'insane' => 0, // raw data unaccepted
28		'entity' => 0, // raw data processed
29		);
30
31
32	1	public function __construct( array $profile =array() )
33		{
34		$defaults = array(
35	1	'model' => 'SampleSchemaThing',
36		'modelOptions' => array(),
37	1	'entityThreshold' => 100, // min numbers of entity that trigger error resilence computation.
38	1	'resilienceToErrors' => 0.3, // if more than 30% of error throws a TooManyErrorException
39	1	'resilienceToInsanes' => 0.9, // if more than 90% of unacceptable data throws a TooManyErrorException
40		'source' => null,
41	1	'datamapper' => function($rawdata){return $rawdata;},
42	1	'dataCleaner' => get_class().'::REMOVE_EMPTY',
43	1	'factsErrorDetector' => get_class().'::NOT_EMPTY_FACTS',
44	1	'rawdataSanitizer' => function($rawdata){return is_array($rawdata)?$rawdata:false;},
45		);
46	1	$this->profile = array_merge($defaults,$profile);
47	1	$this->modelClass = class_exists($this->profile['model'])
48		?$this->profile['model']
49	1	:('\BOTK\Model\\'.$this->profile['model']);
50
51	1	if( !class_exists($this->modelClass) \|\| !is_subclass_of($this->modelClass, '\BOTK\ModelInterface')){
		0 ignored issues – show Bug introduced 2017-02-18 13:26 UTC by Report Bug Copy Issue Report Due to PHP Bug #53727, `is_subclass_of` returns inconsistent results on some PHP versions for interfaces; you could instead use `ReflectionClass::implementsInterface`. Loading history...
52		throw new \InvalidArgumentException("The provided model ({$this->profile['model']} is unknown");
53		}
54	1	if( !is_callable($this->profile['datamapper'])) {
55		throw new \InvalidArgumentException("Invalid datamapper callback");
56		}
57	1	if( !is_callable($this->profile['dataCleaner'])) {
58		throw new \InvalidArgumentException("Invalid dataCleaner callback");
59		}
60	1	if( !is_callable($this->profile['rawdataSanitizer'])) {
61		throw new \InvalidArgumentException("Invalid rawdataSanitizer callback");
62		}
63	1	if( !is_callable($this->profile['factsErrorDetector'])) {
64		throw new \InvalidArgumentException("Invalid factsErrorDetector callback");
65		}
66	1	}
67
68
69		/**
70		* two level filter array, a default for dataCleaner callback
71		*/
72	1	public static function REMOVE_EMPTY( array $data)
73		{
74	1	$a = array();
75	1	foreach ($data as $key => $value) {
76	1	$a[$key] = is_array($value)?array_filter($value):$value;
77		}
78	1	return array_filter($a);
79		}
80
81		/**
82		* a default for dataValidator callback
83		*/
84	1	public static function NOT_EMPTY_FACTS( \BOTK\ModelInterface $data)
85		{
86	1	return $data->getTripleCount()?false:'No facts found.';
87		}
88
89
90		/**
91		* create facts from rawdata. Please nothe that null facts does not means always an error (i.e. no facts is a fact).
92		* if you do not want empty facts use dataValidator
93		*/
94	1	public function factualize($rawData)
95		{
96	1	$rawdataSanitizer = $this->profile['rawdataSanitizer'];
97	1	$validRawData = $rawdataSanitizer($rawData);
98	1	$this->counter['entity']++;
99
100	1	if (!empty($validRawData)){
101	1	$datamapper = $this->profile['datamapper'];
102	1	$dataCleaner = $this->profile['dataCleaner'];
103	1	$factsErrorDetector = $this->profile['factsErrorDetector'];
104	1	$data =$dataCleaner($datamapper($validRawData));
105	1	$facts = call_user_func($this->modelClass.'::fromArray',$data,$this->profile['modelOptions']);
106	1	$this->counter['triple'] += $facts->getTripleCount();
107	1	if($error=$factsErrorDetector($facts)){
108		$this->counter['error']++;
109	1	throw new FactsValidatorWarning($error,$facts);
110		}
111		} else {
112		$this->counter['insane']++;
113		throw new InvalidRawDataWarning("Invalid rawdata",$rawData);
114		}
115
116		// ensure that not too many errors
117	1	$errorRate = ($this->counter['error']/$this->counter['entity']);
118	1	if(( $this->counter['entity'] > $this->profile['entityThreshold'])
119	1	&& ( $errorRate > $this->profile['resilienceToErrors'])){
120		$x = $this->profile['resilienceToErrors']*100;
121		throw new TooManyErrorsException("Error rate in data processing exceeded the $x% threshold");
122		}
123
124		// ensure that not too many insaness raw data
125	1	$insaneRate = ($this->counter['insane']/$this->counter['entity']);
126	1	if(( $this->counter['entity'] > $this->profile['entityThreshold'])
127	1	&& ($insaneRate > $this->profile['resilienceToInsanes'])){
128		$x = $this->profile['resilienceToInsanes']*100;
129		throw new TooManyInsanesException("Unacceptable data rate exceeded the $x% threshold");
130		}
131
132	1	return $facts;
133		}
134
135
136		public function generateLinkedDataHeader()
137		{
138		return call_user_func($this->modelClass.'::getTurtleHeader');
139		}
140
141
142		public function generateLinkedDataFooter()
143		{
144		$now = date('c');
145		$rdf = "
146		#@prefix dct: <http://purl.org/dc/terms/> .
147		#@prefix void: <http://rdfs.org/ns/void#> .
148		#@prefix prov: <http://www.w3.org/ns/prov#> .
149		";
150
151		// add provenance info
152		$rdf .= "#<> prov:generatedAtTime \"$now\"^^xsd:dateTime;";
153		if(!empty($this->profile['source'])){
154		$rdf.= "dct:source <{$this->profile['source']}>;";
155		}
156
157
158		// add dataset info and a human readable comment as last line
159		$rdf.= "foaf:primaryTopic <#dataset>.\n";
160		$rdf.= "#<#dataset> a void:Dataset; void:datadump <>;void:triples {$this->counter['triple']} ;void:entities {$this->counter['entity']}.\n";
161		$rdf.= "#Generated {$this->counter['triple']} good triples from {$this->counter['entity']} entities ({$this->counter['insane']} ignored), {$this->counter['error']} errors\n";
162
163		return $rdf;
164		}
165
166
167		public function addToCounter($counter,$val=1)
168		{
169		if(!array_key_exists($counter,$this->counter)){
170		throw new \InvalidArgumentException("Invalid counter name");
171		}
172		$this->counter[$counter]+= intval($val);
173		}
174
175
176		public function getCounters()
177		{
178		return $this->counter;
179		}
180
181		}

linkeddatacenter / BOTK-core

Pull Request — master (#14)

FactsFactory::REMOVE_EMPTY() A

Complexity

Size

Duplication

Code Coverage

Importance

Duplication Side-by-Side

Filter issues like