diff --git a/content/content.importers.php b/content/content.importers.php index 8e8fd1e..2e1611e 100644 --- a/content/content.importers.php +++ b/content/content.importers.php @@ -31,7 +31,7 @@ public function __construct(){ $this->_driver = Symphony::ExtensionManager()->create('xmlimporter'); } - public function build($context) { + public function build(array $context = array()) { if (isset($context[0])) { if ($context[0] == 'edit' || $context[0] == 'new') { $this->__prepareEdit($context); @@ -49,6 +49,54 @@ public function build($context) { parent::build($context); } + public function addFailedEntries(XMLElement &$fieldset, array $failed_entries) { + foreach ($failed_entries as $index => $current) { + $fieldset->appendChild(new XMLElement( + 'h3', __('Import entry #%d', array($current['position'])) + )); + + // Errors ------------------------------------------------- + + $list = new XMLElement('ol'); + + foreach ($current['errors'] as $error) { + $list->appendChild(new XMLElement('li', $error)); + } + + $fieldset->appendChild($list); + + // Source ------------------------------------------------- + + $entry = $current['element']; + $xml = new DOMDocument(); + $xml->preserveWhiteSpace = false; + $xml->formatOutput = true; + + if(is_null($entry->ownerDocument)) { + $xml->loadXML($entry->saveXML()); + } + else { + $xml->loadXML($entry->ownerDocument->saveXML($entry)); + } + + $source = htmlentities($xml->saveXML($xml->documentElement), ENT_COMPAT, 'UTF-8'); + + $fieldset->appendChild(new XMLElement( + 'pre', "{$source}" + )); + + foreach ($current['values'] as $field => $value) { + if(is_array($value)) $value = implode(',', $value); + $values[$field] = htmlentities($value); + } + + $fieldset->appendChild(new XMLElement( + 'pre', + "" . var_export($values, true) . "" + )); + } + } + /*------------------------------------------------------------------------- Run -------------------------------------------------------------------------*/ @@ -57,9 +105,11 @@ public function __prepareRun($context) { $importManager = new XmlImporterManager(); $html_errors = ini_get('html_errors'); $source = null; + $remote = false; if (isset($_GET['source'])) { $source = $_GET['source']; + $remote = true; } array_shift($context); @@ -73,11 +123,15 @@ public function __prepareRun($context) { continue; } else { - $status = $importer->validate($source); + $status = $importer->validate($source, $remote); } - if ($status == XMLImporter::__OK__) { - $importer->commit(); + if ($_GET['force'] == 'yes') { + $status = XMLImporter::__PARTIAL_OK__; + } + + if (in_array($status, array(XMLImporter::__OK__, XMLImporter::__PARTIAL_OK__))) { + $importer->commit($status); } $this->_runs[] = array( @@ -162,62 +216,74 @@ public function __viewRun() { )) )); - foreach ($failed as $index => $current) { - $fieldset->appendChild(new XMLElement( - 'h3', __('Import entry #%d', array($current['position'])) - )); - - // Errors ------------------------------------------------- - - $list = new XMLElement('ol'); - - foreach ($current['errors'] as $error) { - $list->appendChild(new XMLElement('li', $error)); - } + // Import valid anyway + $button = Widget::Anchor( + __('Import valid entries'), + $this->_uri . '/importers/run/' . $this->_context[1] . '/?force=yes', + __('Import valid entries'), + 'button' + ); - $fieldset->appendChild($list); + $fieldset->appendChild($button); - ### - # Delegate: XMLImporterImportPostRunErrors - # Description: Notify Delegate for Errors - Symphony::ExtensionManager()->notifyMembers( - 'XMLImporterImportPostRunErrors', '/xmlimporter/importers/run/', - array( - $current['errors'] - ) - ); + // Import errors + $fieldset->appendChild(new XMLElement( + 'h3', __('Import Errors') + )); + $this->addFailedEntries($fieldset, $failed); + + ### + # Delegate: XMLImporterImportPostRunErrors + # Description: Notify Delegate for Errors + Symphony::ExtensionManager()->notifyMembers( + 'XMLImporterImportPostRunErrors', '/xmlimporter/importers/run/', + array( + $current['errors'] + ) + ); + } - // Source ------------------------------------------------- + // Invalid entry: + else if ($status == XMLImporter::__PARTIAL_OK__) { + $fieldset->appendChild(new XMLElement( + 'h3', __('Import Partially Complete') + )); - $entry = $current['element']; - $xml = new DOMDocument(); - $xml->preserveWhiteSpace = false; - $xml->formatOutput = true; + // Gather statistics: + $failed = array(); + $importer_result = array( + 'created' => 0, + 'updated' => 0, + 'skipped' => 0, + 'failed' => 0 + ); - if(is_null($entry->ownerDocument)) { - $xml->loadXML($entry->saveXML()); - } - else { - $xml->loadXML($entry->ownerDocument->saveXML($entry)); + foreach ($entries as $index => $current) { + if (!empty($current['errors'])) { + $current['position'] = $index + 1; + $failed[] = $current; + $importer_result++; } - $source = htmlentities($xml->saveXML($xml->documentElement), ENT_COMPAT, 'UTF-8'); + $importer_result[$current['entry']->get('importer_status')]++; + } - $fieldset->appendChild(new XMLElement( - 'pre', "{$source}" - )); + $fieldset->appendChild(new XMLElement( + 'p', __('Import partially completed successfully: %d new entries were created, %d updated, %d were skipped and %d failed.', array( + $importer_result['created'], + $importer_result['updated'], + $importer_result['skipped'], + $importer_result['failed'] + )) + )); - foreach ($current['values'] as $field => $value) { - if(is_array($value)) $value = implode(',', $value); - $values[$field] = htmlentities($value); - } + // Import errors + $fieldset->appendChild(new XMLElement( + 'h3', __('Import Errors') + )); - $fieldset->appendChild(new XMLElement( - 'pre', - "" . var_export($values, true) . "" - )); - } + $this->addFailedEntries($fieldset, $failed); } // Passed: @@ -312,59 +378,34 @@ public function __actionEditNormal() { } else { - // Support {$root} + // Support {$root} and {$workspace} $evaluated_source = str_replace('{$root}', URL, $fields['source']); - if(!filter_var($evaluated_source, FILTER_VALIDATE_URL)) { - $this->_errors['source'] = __('Source is not a valid URL.'); - } - } + $evaluated_source = str_replace('{$workspace}', WORKSPACE, $evaluated_source); - // Namespaces --------------------------------------------------------- - - if ( - isset($fields['discover-namespaces']) - && $fields['discover-namespaces'] == 'yes' - && !isset($this->_errors['source']) - ) { - $gateway = new Gateway(); - $gateway->init(); - $gateway->setopt('URL', $evaluated_source); - $gateway->setopt('TIMEOUT', (int)$fields['timeout']); - $data = $gateway->exec(); - - if ($data === false) { - $this->_errors['discover-namespaces'] = __('Error loading data from URL, make sure it is valid and that it returns data within 60 seconds.'); - } + $ds = DatasourceManager::create($fields['source'], $param_pool, true); + // Not a DataSource (legacy) + if(!($ds instanceof Datasource)) { + if(!filter_var($evaluated_source, FILTER_VALIDATE_URL)) { + $this->_errors['source'] = __('Source is not a valid URL.'); + } + } + // DataSource output else { - preg_match_all('/xmlns:([a-z][a-z-0-9\-]*)="([^\"]+)"/i', $data, $matches); - - if (isset($matches[2][0])) { - $namespaces = array(); - - if (!is_array($fields['namespaces'])) { - $fields['namespaces'] = array(); - } - - foreach ($fields['namespaces'] as $namespace) { - $namespaces[] = $namespace['name']; - $namespaces[] = $namespace['uri']; - } - - foreach ($matches[2] as $index => $uri) { - $name = $matches[1][$index]; - - if (in_array($name, $namespaces) or in_array($uri, $namespaces)) continue; - - $namespaces[] = $name; - $namespaces[] = $uri; + $xml = $ds->execute($param_pool); + if(isset($ds->dsParamNAMESPACES)) { + foreach($ds->dsParamNAMESPACES as $name => $uri) { $fields['namespaces'][] = array( - 'name' => $name, - 'uri' => $uri + 'name' => $name, + 'uri' => $uri ); } } + + if($xml->getAttribute('valid') == 'false') { + $this->_errors['source'] = __('Failed to retrieve data from source: %s', array($xml->generate())); + } } } @@ -583,145 +624,40 @@ public function __viewEdit() { $fieldset = new XMLElement('fieldset'); $fieldset->setAttribute('class', 'settings'); $fieldset->appendChild(new XMLElement('legend', __('Source'))); + $options = array(); - $label = Widget::Label(__('URL')); - $label->appendChild(Widget::Input( - 'fields[source]', General::sanitize( - isset($this->_fields['source']) - ? $this->_fields['source'] - : null - ) - )); - - if (isset($this->_errors['source'])) { - $label = Widget::Error($label, $this->_errors['source']); - } - - $fieldset->appendChild($label); - - $help = new XMLElement('p'); - $help->setAttribute('class', 'help'); - $help->setValue(__('Enter the URL of the XML document you want to process.')); - $fieldset->appendChild($help); - - $label = new XMLElement('p', __('Namespace Declarations')); - $label->setAttribute('class', 'label'); - $fieldset->appendChild($label); - - // Namespaces --------------------------------------------------------- - - $nsFrame = new XMLElement('div'); - $nsFrame->setAttribute('class', 'frame namespaces'); - $namespaces = new XMLElement('ol'); - $namespaces->setAttribute('class', 'namespaces-duplicator'); - $namespaces->setAttribute('data-add', __('Add namespace')); - $namespaces->setAttribute('data-remove', __('Remove namespace')); - - if (isset($this->_fields['namespaces']) and is_array($this->_fields['namespaces'])) { - foreach ($this->_fields['namespaces'] as $index => $data) { - $name = "fields[namespaces][{$index}]"; - - $li = new XMLElement('li'); - $li->appendChild(new XMLElement('header', '

' . __('Namespace') . '

')); - - $group = new XMLElement('div'); - $group->setAttribute('class', 'two columns'); - - $label = Widget::Label(__('Name')); - $label->setAttribute('class', 'column'); - $input = Widget::Input( - "{$name}[name]", - General::sanitize( - isset($data['name']) - ? $data['name'] - : null - ) - ); - $label->appendChild($input); - $group->appendChild($label); - - $label = Widget::Label(__('URI')); - $label->setAttribute('class', 'column'); - $input = Widget::Input( - "{$name}[uri]", - General::sanitize( - isset($data['uri']) - ? $data['uri'] - : null - ) - ); - $label->appendChild($input); - $group->appendChild($label); - - $li->appendChild($group); - $namespaces->appendChild($li); - } - } - - $name = "fields[namespaces][-1]"; - - $li = new XMLElement('li'); - $li->appendChild(new XMLElement('header', '

' . __('Namespace') . '

')); - $li->setAttribute('class', 'template'); - - $input = Widget::Input("{$name}[field]", $field_id, 'hidden'); - $li->appendChild($input); - - $group = new XMLElement('div'); - $group->setAttribute('class', 'two columns'); - - $label = Widget::Label(__('Name')); - $label->setAttribute('class', 'column'); - $input = Widget::Input("{$name}[name]"); - $label->appendChild($input); - $group->appendChild($label); - - $label = Widget::Label(__('URI')); - $label->setAttribute('class', 'column'); - $input = Widget::Input("{$name}[uri]"); - $label->appendChild($input); - $group->appendChild($label); - - $li->appendChild($group); - $namespaces->appendChild($li); - $nsFrame->appendChild($namespaces); - - $fieldset->appendChild($nsFrame); - - // Discover Namespaces ------------------------------------------------ - - $label = Widget::Label(); - $input = Widget::Input('fields[discover-namespaces]', 'yes', 'checkbox'); - - if (!$this->_editing) { - $input->setAttribute('checked', 'checked'); + $datasources = DatasourceManager::listAll(); + foreach($datasources as $index => $ds) { + $options[] = array($ds['handle'], $ds['handle'] == $this->_fields['source'], $ds['name']); } - $label->setValue(__('%s Automatically discover namespaces', array( - $input->generate(false) - ))); + $label = Widget::Label(__('Data Source')); + $label->appendChild(Widget::Select('fields[source]', $options, array('id' => 'ds-context'))); - if (isset($this->_errors['discover-namespaces'])) { - $label = Widget::Error($label, $this->_errors['discover-namespaces']); + if (isset($this->_errors['source'])) { + $label = Widget::Error($label, $this->_errors['source']); } $fieldset->appendChild($label); $help = new XMLElement('p'); $help->setAttribute('class', 'help'); - $help->setValue(__('Search the source document for namespaces, any that it finds will be added to the declarations above.')); + $help->setValue(__('Choose a DataSource that contains the data you wish to import')); $fieldset->appendChild($help); // Included Elements -------------------------------------------------- $label = Widget::Label(__('Included Elements')); - $label->appendChild(Widget::Input( - 'fields[included-elements]', General::sanitize( + $input = Widget::Input( + 'fields[included-elements]', + General::sanitize( isset($this->_fields['included-elements']) ? $this->_fields['included-elements'] : null ) - )); + ); + $input->setAttribute('placeholder', '/data'); + $label->appendChild($input); if (isset($this->_errors['included-elements'])) { $label = Widget::Error($label, $this->_errors['included-elements']); @@ -825,12 +761,14 @@ public function __viewEdit() { $field_id = $field->get('id'); $field_name = "fields[fields][{$index}]"; $field_data = null; + $template_index = null; if (isset($this->_fields['fields'])) { - foreach ($this->_fields['fields'] as $temp_data) { + foreach ($this->_fields['fields'] as $i => $temp_data) { if ($temp_data['field'] != $field_id) continue; $field_data = $temp_data; + $template_index = $i; } } @@ -859,8 +797,8 @@ public function __viewEdit() { ); $label->appendChild($input); - if (isset($this->_errors['fields'][$index])) { - $label = Widget::Error($label, $this->_errors['fields'][$index]); + if (isset($this->_errors['fields'][$template_index])) { + $label = Widget::Error($label, $this->_errors['fields'][$template_index]); } $group->appendChild($label); @@ -999,7 +937,7 @@ public function generateLink($values) { public function __prepareIndex() { $this->_table_columns = array( 'name' => array(__('Name'), true), - 'url' => array(__('URL'), true), + 'source' => array(__('Source'), true), 'elements' => array(__('Included Elements'), true), 'modified' => array(__('Modified'), true), 'author' => array(__('Author'), true) @@ -1139,10 +1077,18 @@ public function __viewIndex() { } else foreach ($this->_importers as $importer) { - $col_name = Widget::TableData(Widget::Anchor( - $importer['about']['name'], - "{$this->_uri}/importers/edit/{$importer['about']['handle']}/" - )); + // Was this importer generated by Version 3, or is it using the legacy method? + if (isset($importer['about']['version'])) { + $col_name = Widget::TableData(Widget::Anchor( + $importer['about']['name'], + "{$this->_uri}/importers/edit/{$importer['about']['handle']}/" + )); + $class = ''; + } else { + $col_name = Widget::TableData($importer['about']['name']); + $class = 'status-notice'; + } + $col_name->appendChild(Widget::Input( "items[{$importer['about']['handle']}]", null, 'checkbox' @@ -1153,9 +1099,17 @@ public function __viewIndex() { )); if (!empty($importer['source'])) { - $col_url = Widget::TableData( - General::sanitize($importer['source']) - ); + $handle = General::sanitize($importer['source']); + $datasources = DatasourceManager::listAll(); + + if(!empty($datasources[$handle]['name'])) { + $source = $datasources[$handle]['name']; + } + else { + $source = $handle; + } + + $col_url = Widget::TableData($source); } else { @@ -1192,7 +1146,7 @@ public function __viewIndex() { $col_name, $col_url, $col_elements, $col_date, $col_author ), - null + $class ); } diff --git a/extension.driver.php b/extension.driver.php index b225b7e..7a41198 100644 --- a/extension.driver.php +++ b/extension.driver.php @@ -198,9 +198,9 @@ public function setXMLImporter(&$name, &$error, $new) { // Options: var_export($new['can-update'], true), - var_export($new['fields'], true), + $this->layoutVar($new['fields']), var_export($new['included-elements'], true), - var_export($new['namespaces'], true), + $this->layoutVar($new['namespaces'], true), var_export($new['source'], true), var_export($new['timeout'], true), var_export($new['section'], true), @@ -221,6 +221,17 @@ public function setXMLImporter(&$name, &$error, $new) { return true; } + private function layoutVar($variable) { + $result = var_export($variable, true); + $result = str_replace(" ", " ", $result); + $result = str_replace("array (", "array(", $result); + $result = str_replace(" => " . PHP_EOL . " ", " => ", $result); + $result = str_replace("," . PHP_EOL . " ),", PHP_EOL . " ),", $result); + $result = str_replace(PHP_EOL, PHP_EOL . " ", $result); + + return $result; + } + public function validateXPath($expression, $namespaces = array()) { $document = new DOMDocument(); $document->loadXML(''); diff --git a/extension.meta.xml b/extension.meta.xml index a4e2aff..cd858f1 100644 --- a/extension.meta.xml +++ b/extension.meta.xml @@ -20,8 +20,15 @@ Brendan Abbott + + Büro für Web- und Textgestaltung + http://hananils.de + + + - Switched from custom data fetching to Data Sources + - Symphony 2.5 support - Fix to allow multiple importers to be selectable on the index again @@ -39,13 +46,13 @@ - Update modification date on import - Fix string handling - + - Added support for the `ImportableField` interface - Added logging for when errors occur - Handle non existent importers better - Update duplicator markup - + - Symphony 2.3 support - Improved localisation support - Ability to use `{$root}` in the URL diff --git a/lib/class.xmlimporter.php b/lib/class.xmlimporter.php index 5eca2d8..9908ee9 100644 --- a/lib/class.xmlimporter.php +++ b/lib/class.xmlimporter.php @@ -18,6 +18,7 @@ class XMLImporter { const __OK__ = 100; + const __PARTIAL_OK__ = 110; const __ERROR_PREPARING__ = 200; const __ERROR_VALIDATING__ = 210; const __ERROR_CREATING__ = 220; @@ -81,6 +82,7 @@ function handleXMLError($errno, $errstr, $errfile, $errline, $context) { $options = $this->options(); $passed = true; + // If $remote, override the source of the XMLImporter with the given $source if ($remote) { if (!is_null($source)) { $options['source'] = $source; @@ -106,8 +108,35 @@ function handleXMLError($errno, $errstr, $errfile, $errline, $context) { } } - else if (!is_null($source)) { - $data = $source; + else if (isset($options['source'])) { + $param_pool = array(); + $ds = DatasourceManager::create($options['source'], $param_pool, true); + + // Not a DataSource (legacy) + if(!($ds instanceof Datasource)) { + $data = $source; + } + // DataSource output + else { + $xml = $ds->execute($param_pool); + + if(isset($ds->dsParamNAMESPACES)) { + foreach($ds->dsParamNAMESPACES as $name => $uri) { + $options['namespaces'][] = array( + 'name' => $name, + 'uri' => $uri + ); + } + } + + if($xml->getAttribute('valid') == 'false') { + $this->_errors[] = __('Failed to retrieve data from source: %s', array($xml->generate())); + $passed = false; + } + else { + $data = '' . $xml->generate(true) . ''; + } + } } else { @@ -216,10 +245,12 @@ function handleXMLError($errno, $errstr, $errfile, $errline, $context) { // Validate: $passed = true; - foreach ($this->_entries as &$current) { + foreach ($this->_entries as $index => &$current) { $entry = EntryManager::create(); $entry->set('section_id', $options['section']); $entry->set('author_id', is_null(Symphony::Engine()->Author()) ? '1' : Symphony::Engine()->Author()->get('id')); + $entry->set('modification_date_gmt', DateTimeObj::getGMT('Y-m-d H:i:s')); + $entry->set('modification_date', DateTimeObj::get('Y-m-d H:i:s')); $values = array(); @@ -254,7 +285,7 @@ function handleXMLError($errno, $errstr, $errfile, $errline, $context) { if ($type == 'author') { if ($field->get('allow_multiple_selection') == 'no') { if(is_array($value)){ - $value = array(implode('', $value)); + $value = array(implode('', $value)); } } } @@ -272,12 +303,20 @@ function handleXMLError($errno, $errstr, $errfile, $errline, $context) { } // Validate: - if (__ENTRY_FIELD_ERROR__ == $entry->checkPostData($values, $current['errors'])) { - $passed = false; - } + try { + if (__ENTRY_FIELD_ERROR__ == $entry->checkPostData($values, $current['errors'])) { + $passed = false; + } - else if (__ENTRY_OK__ != $entry->setDataFromPost($values, $current['errors'], true, true)) { + else if (__ENTRY_OK__ != $entry->setDataFromPost($values, $current['errors'], true, true)) { + $passed = false; + } + } + catch (Exception $ex) { $passed = false; + $current['errors'] = array($ex->getMessage()); + + Symphony::Log()->pushToLog(sprintf('XMLImporter: Failed to set values for entry in position %d, %s', $index, $ex->getMessage()), E_NOTICE, true); } $current['entry'] = $entry; @@ -289,94 +328,95 @@ function handleXMLError($errno, $errstr, $errfile, $errline, $context) { return self::__OK__; } - public function commit() { + public function commit($status) { $options = $this->options(); + $section = SectionManager::fetch($options['section']); $existing = array(); - $section = SectionManager::fetch($options['section']); + // if $status = PARTIAL_OK + if($status == self::__PARTIAL_OK__) { + $entries = $this->_entries; + foreach($entries as $index => $current) { + if(!empty($current['errors'])) { + $this->_entries[$index]['entry']->set('importer_status', 'failed'); + unset($entries[$index]); + } + } + } + else { + $entries = $this->_entries; + } + // Check uniqueness if ((integer)$options['unique-field'] > 0) { $field = FieldManager::fetch($options['unique-field']); - - if (!empty($field)) foreach ($this->_entries as $index => $current) { - $entry = $current['entry']; - - $data = $entry->getData($options['unique-field']); - $where = $joins = $group = null; - - $field->buildDSRetrievalSQL($data, $joins, $where); - - $group = $field->requiresSQLGrouping(); - $entries = EntryManager::fetch(null, $options['section'], 1, null, $where, $joins, $group, false, null, false); - - if (is_array($entries) && !empty($entries)) { - $existing[$index] = $entries[0]['id']; - } - - else { - $existing[$index] = null; - } - } } - foreach ($this->_entries as $index => $current) { + foreach ($entries as $index => $current) { $entry = $current['entry']; $values = $current['values']; $date = DateTimeObj::get('Y-m-d H:i:s'); $dateGMT = DateTimeObj::getGMT('Y-m-d H:i:s'); - $exists = !empty($existing[$index]); - $skip = ($options['can-update'] !== 'yes'); - - // Skip entry - if ($exists && $skip) { - $entry->set('importer_status', 'skipped'); - - ### - # Delegate: XMLImporterEntryPostSkip - # Description: Skipping an entry. Entry object is provided. - Symphony::ExtensionManager()->notifyMembers( - 'XMLImporterEntryPostSkip', '/xmlimporter/importers/run/', - array( - 'section' => $section, - 'entry' => $entry, - 'fields' => $values - ) - ); - } - - // Edit entry - elseif ($exists) { - $entry->set('id', $existing[$index]); - $entry->set('modification_date', $date); - $entry->set('modification_date_gmt', $dateGMT); - - ### - # Delegate: XMLImporterEntryPreEdit - # Description: Just prior to editing of an Entry. - Symphony::ExtensionManager()->notifyMembers( - 'XMLImporterEntryPreEdit', '/xmlimporter/importers/run/', - array( - 'section' => $section, - 'fields' => &$values, - 'entry' => &$entry - ) - ); - - EntryManager::edit($entry); - $entry->set('importer_status', 'updated'); + // Uniqueness check (if required) + if(!empty($field)) { + $this->checkExisting($field, $entry, $index, $existing); + }; + + // Matches an existing entry + if (!is_null($existing[$index])) { + // Update + if ($options['can-update'] == 'yes') { + $entry->set('id', $existing[$index]); + $entry->set('modification_date', $date); + $entry->set('modification_date_gmt', $dateGMT); + + ### + # Delegate: XMLImporterEntryPreEdit + # Description: Just prior to editing of an Entry. + Symphony::ExtensionManager()->notifyMembers( + 'XMLImporterEntryPreEdit', '/xmlimporter/importers/run/', + array( + 'section' => $section, + 'fields' => &$values, + 'entry' => &$entry + ) + ); + + EntryManager::edit($entry); + $entry->set('importer_status', 'updated'); + + ### + # Delegate: XMLImporterEntryPostEdit + # Description: Editing an entry. Entry object is provided. + Symphony::ExtensionManager()->notifyMembers( + 'XMLImporterEntryPostEdit', '/xmlimporter/importers/run/', + array( + 'section' => $section, + 'entry' => $entry, + 'fields' => $values + ) + ); + } - ### - # Delegate: XMLImporterEntryPostEdit - # Description: Editing an entry. Entry object is provided. - Symphony::ExtensionManager()->notifyMembers( - 'XMLImporterEntryPostEdit', '/xmlimporter/importers/run/', - array( - 'section' => $section, - 'entry' => $entry, - 'fields' => $values - ) - ); + // Skip + else { + $entry->set('importer_status', 'skipped'); + + ### + # Delegate: XMLImporterEntryPostSkip + # Description: Skipping an entry. Entry object is provided. + Symphony::ExtensionManager()->notifyMembers( + 'XMLImporterEntryPostSkip', '/xmlimporter/importers/run/', + array( + 'section' => $section, + 'entry' => $entry, + 'fields' => $values + ) + ); + + continue; + } } // Create entry @@ -415,4 +455,40 @@ public function commit() { } } } + + /** + * Given the `$field`, and the `$entry`, this function + * will take the value that is about to be imported and + * check to see if it's already in the system. + * If it is, the `entry_id` of `$entry` will be added + * to the `$existing` array. + * + * @param Field $field + * The unique field + * @param Entry $entry + * The current entry that is about to be imported + * @param integer $index + * The current position of the Entry in the import + * @param array $existing + * An associative array, by reference. The key is the position of + * the entry in the import, and the value is the `entry_id` if + * a match was found, otherwise null. + */ + private function checkExisting(Field $field, Entry $entry, $index, array &$existing) { + $data = $entry->getData($field->get('id')); + $where = $joins = $group = null; + + $field->buildDSRetrievalSQL($data, $joins, $where); + + $group = $field->requiresSQLGrouping(); + $existing_entries = EntryManager::fetch(null, $field->get('parent_section'), 1, null, $where, $joins, $group, false, null, false); + + if (is_array($existing_entries) && !empty($existing_entries)) { + $existing[$index] = $existing_entries[0]['id']; + } + + else { + $existing[$index] = null; + } + } } diff --git a/lib/class.xmlimportermanager.php b/lib/class.xmlimportermanager.php index f974d4f..64d4d65 100644 --- a/lib/class.xmlimportermanager.php +++ b/lib/class.xmlimportermanager.php @@ -66,7 +66,8 @@ public function about($name){ $handle = $this->__getHandleFromFilename(basename($path)); if(is_callable(array($classname, 'about'))){ - $about = call_user_func(array($classname, 'about')); + $importer = new $classname; + $about = $importer->about(); return array_merge($about, array('handle' => $handle)); } diff --git a/lib/markdownify/markdownify.php b/lib/markdownify/markdownify.php index 43730cb..7514f6d 100644 --- a/lib/markdownify/markdownify.php +++ b/lib/markdownify/markdownify.php @@ -321,13 +321,13 @@ function parse() { } break; default: - trigger_error('invalid node type', E_USER_ERROR); + //trigger_error('invalid node type', E_USER_ERROR); break; } $this->lastWasBlockTag = $this->parser->nodeType == 'tag' && $this->parser->isStartTag && $this->parser->isBlockElement; } if (!empty($this->buffer)) { - trigger_error('buffer was not flushed, this is a bug. please report!', E_USER_WARNING); + //trigger_error('buffer was not flushed, this is a bug. please report!', E_USER_WARNING); while (!empty($this->buffer)) { $this->out($this->unbuffer()); } @@ -932,7 +932,7 @@ function stack() { */ function unstack() { if (!isset($this->stack[$this->parser->tagName]) || !is_array($this->stack[$this->parser->tagName])) { - trigger_error('Trying to unstack from empty stack. This must not happen.', E_USER_ERROR); + //trigger_error('Trying to unstack from empty stack. This must not happen.', E_USER_ERROR); } return array_pop($this->stack[$this->parser->tagName]); } diff --git a/readme.md b/readme.md index 94a317b..a5bc68c 100644 --- a/readme.md +++ b/readme.md @@ -2,92 +2,26 @@ XML Importer is a way of creating repeatable templates to import data from XML feeds directly into Symphony sections. It provides a way of mapping content from XML nodes directly onto fields in your sections, and the ability to both create new and update existing entries. -## Installation +## Sources -1. Upload the `xmlimporter` folder to your Symphony `/extensions` folder -2. Enable it by selecting "XML Importer" on the System > Extensions page, choose Enable from the with-selected menu, then click Apply -3. Use the extension from the Blueprints > XML Importers menu +As of version 3, XML Importer uses Data Sources to import data: +- If you'd like to import external data, please have a look at [Remote Data Source]() which accepts XML, JSON or CSV sources. +- If you'd like to alter existing data, you can make use of standard section Data Sources. -## Creating an Importer (tutorial) -An Importer is similar to a Dynamic XML Datasource in its configuration. Let's create a fictitious importer to store your Twitter messages into a section named "Tweets" that has three fields: +If you want to use the same importer for different input sources, you can modify the URL of the run: -* Permalink (Text Input) -* Date (Date) -* Tweet (Textarea) - - -### Essentials - -Start by creating a new Importer and give it a sensible **Name** such as `Tweets` and add any notes into the **Description** field: `Import Tweets from user's public RSS timeline`. - - -### Source - -This is where we define the XML feed. Start by providing the feed URL, for example mine is: - - http://www.twitter.com/statuses/user_timeline/12675.rss - -Take a look at the source of this feed and you'll see that it uses XML namespaces (the `xmlns` attributes). To be able to traverse this feed properly the XML Importer needs to know about these namespaces, so click **Add item** under the **Namespace Declarations** region to add the Name/URI values for each namespace: - -* Name: `rss`, URI: `http://www.w3.org/2005/Atom` -* Name: `georss`, URI: `http://www.georss.org/georss` - -**Included Elements** is an XPath expression representing each XML node that you want to convert into a Symphony entry. In our example we want to loop over each `` node in the RSS feed: - - /rss/channel/item - -This can also be written as: - - //item - - -### Destination - -Now we configure the values for each field in our new entry. Start by selecting the section into which we want to create new entries (`Tweets`). The dropdown under **Fields** will now be filled with the name of each field in this section. We are going to store values for each of the Permalink, Date and Tweet fields. - -Click **Add item** to configure the `Permalink` field. You will see three options appear: **XPath Expression**, **PHP Function** and **Is unique**. - -Since our **Included Elements** are going to be `` elements from the RSS feed, here is an example: - - - - -The **XPath Expression** for the `Permalink` field is therefore going to be relative to an `` element. To get the tweet text the XPath would be: - - permalink/text() - -Remember we want the text value (`text()`) and not the element itself! - -**PHP Function** can be left blank, but is used for more complex processing of the selected value before saving. Be sure to check the **Is unique** radio button for the `Permalink` field so that duplicates cannot be created — this prevents the same tweets being added every time the importer is run. - -Repeat the above step for the remaining fields. Your **Fields** options should eventually look like: - -* **Permalink** XPath Expression: `permalink/text()`, PHP Function: (blank), Is unique: `Yes` -* **Date**, XPath Expression: `published/text()`, PHP Function: (blank), Is unique: `No` -* **Description**, XPath Expression: `description/text()`, PHP Function: (blank), Is unique: `No` - -Finally, untick the **Can update existing entries** checkbox. When used in conjunction with the **Is unique** selection, this option allows the importer to update entries where the "unique" value matches. Twitter doesn't allow you edit Tweets once published, so this option isn't required. - -Save your importer. - - -## Run an Importer - -From the XML Importers list, click a row to highlight the importer and select **Run** from the **With Selected...** dropdown. - -If you want to use the same Importer in multiple feeds (if you have more than one Twitter feed, for example) you can modify the URL of the Run URL. By default our Twitter importer will be executed this this URL: - - /symphony/extension/xmlimporter/importers/run/twitter/ + http://example.com/symphony/extension/xmlimporter/importers/run/twitter/?source=http://twitter.com/statuses/public_timeline.rss -But the feed URL can be overridden by appending a `source` parameter: +## Destinations - http://example.com/symphony/extension/xmlimporter/importers/run/twitter/?source=http://twitter.com/statuses/public_timeline.rss +Imported data will be stored in the section of your choice (either creating new entries or updating existing ones based on your importer's settings). +When setting up **XPath expressions**, please keep in mind that you explicitely have to reference the text node to get the value of element, e. g. `example/text()`. ## Using PHP Functions -The **PHP Function** field on each imported field allows for some additional processing of the value returned from the XPath Expression. This might be used for parsing out an image URL, converting to Markdown, formatting a date, and so on. +The PHP Function field on each imported field allows for some additional processing of the value returned from the XPath Expression. This might be used for parsing out an image URL, converting to Markdown, formatting a date, and so on. For example, if you wanted to store a hash of the Permalink instead of the URL, you can use the following: @@ -109,6 +43,4 @@ Since XMLImporter 2.1, the `prepareImportValue` function will choose the first m ## Advanced Tips -- The `{$root}` parameter can used in your source URL which will be evaluated at run time to the value of the Symphony constant URL, eg. `{$root}/feed/news/` becomes `http://example.com/feed/news/` -- By default Symphony will set the timeout for retrieving the source URL to be 60 seconds. This can be updated by modifying the `timeout` setting in your saved XML Importer file which are located in `/workspace/xml-importers/`. - +By default Symphony will set the timeout for retrieving the source URL to be 60 seconds. This can be updated by modifying the `timeout` setting in your saved XML Importer file which are located in `/workspace/xml-importers/`. diff --git a/templates/xml-importer.php b/templates/xml-importer.php index 0f8550f..f9f4ead 100644 --- a/templates/xml-importer.php +++ b/templates/xml-importer.php @@ -1,37 +1,40 @@ %s, - 'author' => array( - 'name' => %s, - 'email' => %s - ), - 'description' => %s, - 'file' => __FILE__, - 'created' => %s, - 'updated' => %s - ); - } +class XMLImporter%s extends XMLImporter { + public function about() + { + return array( + 'name' => %s, + 'author' => array( + 'name' => %s, + 'email' => %s + ), + 'description' => %s, + 'file' => __FILE__, + 'created' => %s, + 'updated' => %s, + 'version' => 'XML Importer 3.0' + ); + } - public function options() { - return array( - 'can-update' => %s, - 'fields' => %s, - 'included-elements' => %s, - 'namespaces' => %s, - 'source' => %s, - 'timeout' => %s, - 'section' => %s, - 'unique-field' => %s - ); - } - - public function allowEditorToParse() { - return true; - } - } + public function options() + { + return array( + 'can-update' => %s, + 'fields' => %s, + 'included-elements' => %s, + 'namespaces' => %s, + 'source' => %s, + 'timeout' => %s, + 'section' => %s, + 'unique-field' => %s + ); + } + public function allowEditorToParse() + { + return true; + } +}