Skip to content

Commit

Permalink
Merge branch 'master' of github.com:symphonycms/remote_datasource
Browse files Browse the repository at this point in the history
  • Loading branch information
@nitriques
nitriques committed May 13, 2015
2 parents 97e5182 + 8056ffe commit d896f20
Show file tree
Hide file tree
Showing 9 changed files with 267 additions and 98 deletions.
12 changes: 11 additions & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# Remote Datasource

#### Version 2.1
#### Version 2.1.2

The Remote Datasource allows you to consume XML, JSON, CSV and TXT sources in Symphony. This extension aims to build upon the Dynamic XML datasource functionality provided in Symphony to allow better cache control, the automatic discovery of namespaces and more flexibility.

Expand Down Expand Up @@ -35,4 +35,14 @@ public static function prepareGateway(&$gateway) {}
* the parsed xml string data returned by the Gateway by reference
*/
public function exposeData(&$data) {}

/**
* This method is called when their is an http error
* or when content type is unsupported
*
* @since Remote Datasource 2.0
* @param array $info
* info of the http request
*/
public function httpError(&$info) {}
````
190 changes: 105 additions & 85 deletions data-sources/datasource.remote.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,8 @@
class RemoteDatasource extends DataSource implements iDatasource
{

private static $transformer = null;
private static $url_result = null;

private static $cacheable = null;

public static function getName()
Expand Down Expand Up @@ -71,6 +71,19 @@ public function exposeData(&$data)

}

/**
* This method is called when their is an http error
* or when content type is unsupported
*
* @since Remote Datasource 2.0
* @param array $info
* info of the http request
*/
public function httpError(&$info)
{

}

/*-------------------------------------------------------------------------
Utilities
-------------------------------------------------------------------------*/
Expand Down Expand Up @@ -119,23 +132,7 @@ public static function isValidURL($url, $timeout = 6, $format = 'xml', $fetch_UR
if (trim($url) == '') {
return __('This is a required field');
} elseif ($fetch_URL === true) {
$gateway = new Gateway;
$gateway->init($url);
$gateway->setopt('TIMEOUT', $timeout);

// Set the approtiate Accept: headers depending on the format of the URL.
if ($format == 'xml') {
$gateway->setopt('HTTPHEADER', array('Accept: text/xml, */*'));
} elseif ($format == 'json') {
$gateway->setopt('HTTPHEADER', array('Accept: application/json, */*'));
} elseif ($format == 'csv') {
$gateway->setopt('HTTPHEADER', array('Accept: text/csv, */*'));
}

self::prepareGateway($gateway);

$data = $gateway->exec();
$info = $gateway->getInfoLast();
list($data, $info) = self::fetch($url, $format, $timeout);

// 28 is CURLE_OPERATION_TIMEOUTED
if (isset($info['curl_error']) && $info['curl_error'] == 28) {
Expand Down Expand Up @@ -226,7 +223,7 @@ public static function buildCacheID($settings)
*/
public static function buildCacheInformation(XMLElement $wrapper, Cacheable $cache, $cache_id)
{
$cachedData = $cache->check($cache_id);
$cachedData = $cache->read($cache_id);

if (is_array($cachedData) && !empty($cachedData) && (time() < $cachedData['expiry'])) {
$a = Widget::Anchor(__('Clear now'), SYMPHONY_URL . getCurrentPage() . 'clear_cache/');
Expand Down Expand Up @@ -256,7 +253,7 @@ public static function buildEditor(XMLElement $wrapper, array &$errors = array()

// If `clear_cache` is set, clear it..
if (isset($cache_id) && in_array('clear_cache', Administration::instance()->Page->getContext())) {
$cache->forceExpiry($cache_id);
$cache->delete($cache_id);
Administration::instance()->Page->pageAlert(
__('Data source cache cleared at %s.', array(Widget::Time()->generate()))
. '<a href="' . SYMPHONY_URL . '/blueprints/datasources/" accesskey="a">'
Expand Down Expand Up @@ -565,7 +562,8 @@ public static function prepare(array $settings, array $params, $template)
$settings['namespaces'] = $namespaces;
$cache = Symphony::ExtensionManager()->getCacheProvider('remotedatasource');
$cache_id = self::buildCacheID($settings);
$cache->write($cache_id, self::$url_result, $settings['cache']);
$data = self::transformResult(self::$url_result, $settings['format']);
$cache->write($cache_id, $data, $settings['cache']);
}

return sprintf(
Expand Down Expand Up @@ -639,7 +637,7 @@ public function execute(array &$param_pool = null)
// Check for an existing Cache for this Datasource
$cache_id = self::buildCacheID($this);
$cache = Symphony::ExtensionManager()->getCacheProvider('remotedatasource');
$cachedData = $cache->check($cache_id);
$cachedData = $cache->read($cache_id);
$writeToCache = null;
$isCacheValid = true;
$creation = DateTimeObj::get('c');
Expand All @@ -650,34 +648,16 @@ public function execute(array &$param_pool = null)
|| (time() - $cachedData['creation']) > ($this->dsParamCACHE * 60) // The cache is old.
) {
if (Mutex::acquire($cache_id, $this->dsParamTIMEOUT, TMP)) {
$ch = new Gateway;
$ch->init($this->dsParamURL);
$ch->setopt('TIMEOUT', $this->dsParamTIMEOUT);

// Set the approtiate Accept: headers depending on the format of the URL.
if ($this->dsParamFORMAT == 'xml') {
$ch->setopt('HTTPHEADER', array('Accept: text/xml, */*'));
} elseif ($this->dsParamFORMAT == 'json') {
$ch->setopt('HTTPHEADER', array('Accept: application/json, */*'));
} elseif ($this->dsParamFORMAT == 'csv') {
$ch->setopt('HTTPHEADER', array('Accept: text/csv, */*'));
}

self::prepareGateway($ch);

$data = $ch->exec();
$info = $ch->getInfoLast();

list($data, $info) = self::fetch($this->dsParamURL, $this->dsParamFORMAT, $this->dsParamTIMEOUT);
Mutex::release($cache_id, TMP);

$data = trim($data);
$writeToCache = true;

// Handle any response that is not a 200, or the content type does not include XML, JSON, plain or text
if ((int) $info['http_code'] != 200 || !preg_match('/(xml|json|csv|plain|text)/i', $info['content_type'])) {
$writeToCache = false;

$result->setAttribute('valid', 'false');
$this->httpError($info);

// 28 is CURLE_OPERATION_TIMEOUTED
if ($info['curl_error'] == 28) {
Expand All @@ -697,76 +677,47 @@ public function execute(array &$param_pool = null)
}

return $result;
} else if (strlen($data) > 0) {

// Handle where there is `$data`

// If it's JSON, convert it to XML
if ($this->dsParamFORMAT == 'json') {
try {
require_once TOOLKIT . '/class.json.php';
$data = JSON::convertToXML($data);
} catch (Exception $ex) {
$writeToCache = false;
$errors = array(
array('message' => $ex->getMessage())
);
}
} elseif ($this->dsParamFORMAT == 'csv') {
try {
require_once EXTENSIONS . '/remote_datasource/lib/class.csv.php';
$data = CSV::convertToXML($data);
} catch (Exception $ex) {
$writeToCache = false;
$errors = array(
array('message' => $ex->getMessage())
);
}
} elseif ($this->dsParamFORMAT == 'txt') {
$txtElement = new XMLElement('entry');
$txtElement->setValue(General::wrapInCDATA($data));
$data = $txtElement->generate();
$txtElement = null;
}
else if (!General::validateXML($data, $errors, false, new XsltProcess)) {
// If the XML doesn't validate..
$writeToCache = false;
// Handle where there is `$data`
} else if (strlen($data) > 0) {
try {
$data = self::transformResult($data, $this->dsParamFORMAT);
}

// If the `$data` is invalid, return a result explaining why
if ($writeToCache === false) {
catch (TransformException $ex) {
$writeToCache = false;
$error = new XMLElement('errors');
$error->setAttribute('valid', 'false');

$error->appendChild(new XMLElement('error', __('Data returned is invalid.')));

foreach ($errors as $e) {
foreach ($ex->getErrors() as $e) {
if (strlen(trim($e['message'])) == 0) {
continue;
}

$error->appendChild(new XMLElement('item', General::sanitize($e['message'])));
}

$result->appendChild($error);

return $result;
}

// If `$data` is empty, set the `force_empty_result` to true.
} elseif (strlen($data) == 0) {

// If `$data` is empty, set the `force_empty_result` to true.
$this->_force_empty_result = true;
}

// Failed to acquire a lock
} else {

// Failed to acquire a lock
$result->appendChild(
new XMLElement('error', __('The %s class failed to acquire a lock.', array('<code>Mutex</code>')))
);
}

// The cache is good, use it!
} else {

// The cache is good, use it!
$data = trim($cachedData['data']);
$creation = DateTimeObj::get('c', $cachedData['creation']);
}
Expand Down Expand Up @@ -832,6 +783,75 @@ public function execute(array &$param_pool = null)

return $result;
}

/**
* Given a URL, Format and Timeout, this function will initalise
* Symphony's Gateway class to retrieve the contents of the URL.
*
* @param string $url
* @param string $format
* @param integer $timeout
* @return array
*/
public static function fetch($url, $format, $timeout)
{
$ch = new Gateway;
$ch->init($url);
$ch->setopt('TIMEOUT', $timeout);

// Set the approtiate Accept: headers depending on the format of the URL.
if ($transformer = self::getTransformer($format)) {
$accepts = $transformer->accepts();
$ch->setopt('HTTPHEADER', array('Accept: ' . $accepts));
}

self::prepareGateway($ch);

return array(
trim($ch->exec()),
$ch->getInfoLast()
);
}

/**
* Given the result (a string), and a desired format, this
* function will transform it to the desired format and return
* it.
* @param string $data
* @param string $format
* @return string
*/
public static function transformResult($data, $format)
{
if ($transformer = self::getTransformer($format)) {
$data = $transformer->transform($data);
} else {
$data = '';
}

return $data;
}

/**
* Given the format, this function will look for the file
* and create a new transformer.
*
* @param string $format
* @return Transformer
*/
public static function getTransformer($format)
{
$transformer = EXTENSIONS . '/remote_datasource/lib/class.' . strtolower($format) . '.php';

if (!isset(self::$transformer)) {
if (file_exists($transformer)) {
$classname = require_once $transformer;
self::$transformer = new $classname;
}
}

return self::$transformer;
}
}

return 'RemoteDatasource';
8 changes: 8 additions & 0 deletions extension.meta.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,14 @@
</author>
</authors>
<releases>
<release version="2.1.2" date="2014-12-17" min="2.4" max="2.6.x">
- Fix the first cache of a result always resulting in an error
- Abstract the transformers for better extensibility
</release>
<release version="2.1.1" date="2014-09-28" min="2.4">
- Expose the CURL error via `httpError()`
- Fix error with CSV importing
</release>
<release version="2.1.0" date="2014-06-25" min="2.4">
- Add support for text format (a copy of the html response body)
- Add some documentation
Expand Down
44 changes: 32 additions & 12 deletions lib/class.csv.php
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,27 @@
<?php

class CSV
require_once __DIR__ . '/interface.transformer.php';
require_once __DIR__ . '/class.transformexception.php';

class CSVFormatter implements Transformer
{
public function accepts()
{
return 'text/csv, */*';
}

public function transform($data)
{
try {
$data = self::convertToXML($data);
} catch (Exception $ex) {
throw new TransformException($ex->getMessage(), array(
'message' => $ex->getMessages()
));
}

return $data;
}

/**
* Given a CSV file, generate a resulting XML tree
Expand Down Expand Up @@ -50,19 +70,19 @@ public static function convertToXML($data)
*/
public static function addRow(DOMDocument $doc, DOMElement $root, $row, $headers)
{
foreach ($row as $column) {
// Create <entry><header>value</header></entry>
$entry = $doc->createElement('entry');

foreach ($headers as $i => $header) {
$col = $doc->createElement($header);
$col = $entry->appendChild($col);
// Create <entry><header>value</header></entry>
$entry = $doc->createElement('entry');

$value = $doc->createTextNode($row[$i]);
$value = $col->appendChild($value);
}
foreach ($headers as $i => $header) {
$col = $doc->createElement($header);
$col = $entry->appendChild($col);

$root->appendChild($entry);
$value = $doc->createTextNode($row[$i]);
$value = $col->appendChild($value);
}

$root->appendChild($entry);
}
}

return 'CSVFormatter';
Loading

0 comments on commit d896f20

Please sign in to comment.