Implementation of MapBox vector tiles (MBTiles pbf) serving #42

Faster check for grids in large mbtiles file. Closes #51.

README.md

@@ -1,5 +1,5 @@
-TileServer PHP - OGC Web Map Tiling Server (WMTS)
-=================================================
+TileServer PHP: MapTiler and MBTiles maps via WMTS
+==================================================
 
 This server distributes maps to desktop, web, and mobile applications from
 a standard Apache+PHP web hosting.
@@ -111,15 +111,11 @@ Supported protocols:
   Exposed at http://[...]/layer/z/x/y.grid.json
   
 
-To use the OGC WMTS services point your client (desktop or web) to the URL
+To use the OGC WMTS standard point your client (desktop or web) to the URL
 of 'directory' where you installed tileserver.php project with suffix "wmts".
-
 For example: http://www.example.com/directory/wmts
- 
-You can also install the project into a root directory of a virtual server:
-Then the address is: http://www.example.com/wmts
 
-Similarly for another end points.
+If you have installed the project into a root directory of a domain, then the address is: http://www.example.com/wmts
 
 The supported WMTS requests includes:
 
@@ -133,7 +129,7 @@ GetTile RESTful/KVP:
    http://[...]/layer/[ANYTHING-OPTIONAL][z]/[x]/[y].[ext]
    http://[...]?service=wmts&request=getTile&layer=[layer]&tilematrix=[z]&tilerow=[y]&tilecol=[y]&format=[ext]
    
-Another example requests are mentioned in the .htaccess.
+Other example requests are mentioned in the .htaccess.
 
 Performance from the web clients
 --------------------------------
@@ -180,10 +176,10 @@ Password protection
 HTTP Simple Authentication can be easily added to the server.
 Edit the .htaccess and add these lines:
 
-AuthUserFile /full/path/to/.htpasswd
-AuthType Basic
-AuthName "Secure WMTS"
-Require valid-user
+    AuthUserFile /full/path/to/.htpasswd
+    AuthType Basic
+    AuthName "Secure WMTS"
+    Require valid-user
 
 Create a file called .htpasswd with user:password format.
 You can use a command-line utility:
@@ -240,7 +236,7 @@ Tested WMTS/TMS clients
 BSD License
 -----------
 
-Copyright (C) 2012 Klokan Technologies GmbH
+Copyright (C) 2015 Klokan Technologies GmbH (http://www.klokantech.com/)
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without

tileserver.php

@@ -71,13 +71,13 @@ class Server {
     $mjs = glob('*/metadata.json');
     $mbts = glob('*.mbtiles');
     if ($mjs) {
-      foreach ($mjs as $mj) {
+      foreach (array_filter($mjs, 'is_readable') as $mj) {
         $layer = $this->metadataFromMetadataJson($mj);
         array_push($this->fileLayer, $layer);
       }
     }
     if ($mbts) {
-      foreach ($mbts as $mbt) {
+      foreach (array_filter($mbts, 'is_readable') as $mbt) {
         $this->dbLayer[] = $this->metadataFromMbtiles($mbt);
       }
     }
@@ -167,13 +167,54 @@ class Server {
 
     $resultdata = $result->fetchAll();
     foreach ($resultdata as $r) {
-      $metadata[$r['name']] = $r['value'];
+      $value = preg_replace('/(\\n)+/','',$r['value']); 
+      $metadata[$r['name']] = addslashes($value);
+    }
+    if (!array_key_exists('minzoom', $metadata)
+    || !array_key_exists('maxzoom', $metadata)
+    ) {
+      // autodetect minzoom and maxzoom
+      $result = $this->db->query('select min(zoom_level) as min, max(zoom_level) as max from tiles');
+      $resultdata = $result->fetchAll();
+      if (!array_key_exists('minzoom', $metadata))
+        $metadata['minzoom'] = $resultdata[0]['min'];
+      if (!array_key_exists('maxzoom', $metadata))
+        $metadata['maxzoom'] = $resultdata[0]['max'];
+    }
+    // autodetect format using JPEG magic number FFD8
+    if (!array_key_exists('format', $metadata)) {
+      $result = $this->db->query('select hex(substr(tile_data,1,2)) as magic from tiles limit 1');
+      $resultdata = $result->fetchAll();
+      $metadata['format'] = ($resultdata[0]['magic'] == 'FFD8')
+        ? 'jpg'
+        : 'png';
+    }
+    // autodetect bounds
+    if (!array_key_exists('bounds', $metadata)) {
+      $result = $this->db->query('select min(tile_column) as w, max(tile_column) as e, min(tile_row) as s, max(tile_row) as n from tiles where zoom_level='.$metadata['maxzoom']);
+      $resultdata = $result->fetchAll();
+      $w = -180 + 360 * ($resultdata[0]['w'] / pow(2,$metadata['maxzoom']));
+      $e = -180 + 360 * ((1+$resultdata[0]['e']) / pow(2,$metadata['maxzoom']));
+      $n = $this->row2lat($resultdata[0]['n'], $metadata['maxzoom']);
+      $s = $this->row2lat($resultdata[0]['s']-1, $metadata['maxzoom']);
+      $metadata['bounds'] = implode(',', array($w, $s, $e, $n));
     }
     $metadata = $this->metadataValidation($metadata);
     $mbt = explode('.', $mbt);
     $metadata['basename'] = $mbt[0];
     return $metadata;
   }
+  
+  /**
+   * Convert row number to latitude of the top of the row
+   * @param integer $r
+   * @param integer $zoom
+   * @return integer
+   */
+   public function row2lat($r, $zoom) {
+     $y = $r / pow(2,$zoom-1) - 1;
+     return rad2deg(2.0 * atan(exp(3.191459196*$y)) - 1.57079632679489661922);
+   }
 
   /**
    * Valids metaJSON
@@ -182,7 +223,6 @@ class Server {
    */
   public function metadataValidation($metadata) {
     if (array_key_exists('bounds', $metadata)) {
-// TODO: Calculate bounds from tiles if bounds is missing - with GlobalMercator
       $metadata['bounds'] = array_map('floatval', explode(',', $metadata['bounds']));
     } else {
       $metadata['bounds'] = array(-180, -85.051128779807, 180, 85.051128779807);
@@ -190,8 +230,7 @@ class Server {
     if (!array_key_exists('profile', $metadata)) {
       $metadata['profile'] = 'mercator';
     }
-// TODO: detect format, minzoom, maxzoom, thumb
-// scandir() for directory / SQL for mbtiles
+// TODO: detect thumb / SQL for mbtiles
     if (array_key_exists('minzoom', $metadata))
       $metadata['minzoom'] = intval($metadata['minzoom']);
     else
@@ -203,11 +242,6 @@ class Server {
     if (!array_key_exists('format', $metadata)) {
       $metadata['format'] = 'png';
     }
-    /*
-      if (!array_key_exists('thumb', $metadata )) {
-      $metadata['profile'] = 'mercator';
-      }
-     */
     return $metadata;
   }
 
@@ -257,9 +291,10 @@ class Server {
    * @param integer $x
    * @param string $ext
    */
-  public function getTile($tileset, $z, $y, $x, $ext) {
+  public function renderTile($tileset, $z, $y, $x, $ext) {
     if ($this->isDBLayer($tileset)) {
       if ($this->isModified($tileset) == TRUE) {
+        header('Access-Control-Allow-Origin: *');
         header('HTTP/1.1 304 Not Modified');
         die;
       }
@@ -274,7 +309,11 @@ class Server {
       $result = $this->db->query('select tile_data as t from tiles where zoom_level=' . $z . ' and tile_column=' . $x . ' and tile_row=' . $y);
       $data = $result->fetchColumn();
       if (!isset($data) || $data === FALSE) {
-        $this->getCleanTile();
+        //scale of tile (for retina tiles)
+        $result = $this->db->query('select value from metadata where name="scale"');
+        $resultdata = $result->fetchColumn();
+        $scale = isset($resultdata) && $resultdata !== FALSE ? $resultdata : 1;
+        $this->getCleanTile($scale);
       } else {
         $result = $this->db->query('select value from metadata where name="format"');
         $resultdata = $result->fetchColumn();
@@ -282,33 +321,55 @@ class Server {
         if ($format == 'jpg') {
           $format = 'jpeg';
         }
-        header('Content-type: image/' . $format);
+        if ($format == 'pbf') {
+          header('Content-type: application/x-protobuf');
+          header('Content-Encoding:gzip');
+        } else {
+          header('Content-type: image/' . $format);
+        }
+        header('Access-Control-Allow-Origin: *');
         echo $data;
       }
     } elseif ($this->isFileLayer($tileset)) {
       $name = './' . $tileset . '/' . $z . '/' . $x . '/' . $y . '.' . $ext;
       if ($fp = @fopen($name, 'rb')) {
+        header('Access-Control-Allow-Origin: *');
         header('Content-Type: image/' . $ext);
         header('Content-Length: ' . filesize($name));
         fpassthru($fp);
         die;
       } else {
-        $this->getCleanTile();
+        //scale of tile (for retina tiles)
+        $meta = json_decode(file_get_contents($tileset.'/metadata.json'));
+        if(!isset($meta->scale)){
+          $meta->scale = 1;
+        }
+        if ($ext == 'pbf') {
+          header('HTTP/1.1 404 Not Found');
+          header('Content-Type: application/json; charset=utf-8');
+          echo '{"message":"Tile does not exist"}';
+          die;
+        }
+        $this->getCleanTile($meta->scale);
       }
     } else {
-      echo 'Server: Unknown or not specified dataset';
+      header('HTTP/1.1 404 Not Found');
+      echo 'Server: Unknown or not specified dataset "'.$tileset.'"';
       die;
     }
   }
 
   /**
    * Returns clean tile
+   * @param integer $scale Default 1
    */
-  public function getCleanTile() {
-    $png = imagecreatetruecolor(256, 256);
+  public function getCleanTile($scale = 1) {
+    $tileSize = 256 * $scale;
+    $png = imagecreatetruecolor($tileSize, $tileSize);
     imagesavealpha($png, true);
     $trans_colour = imagecolorallocatealpha($png, 0, 0, 0, 127);
     imagefill($png, 0, 0, $trans_colour);
+    header('Access-Control-Allow-Origin: *');
     header('Content-type: image/png');
     imagepng($png);
     die;
@@ -321,7 +382,7 @@ class Server {
    * @param integer $y
    * @param integer $x
    */
-  public function getUTFGrid($tileset, $z, $y, $x, $flip = TRUE) {
+  public function renderUTFGrid($tileset, $z, $y, $x, $flip = TRUE) {
     if ($this->isDBLayer($tileset)) {
       if ($this->isModified($tileset) == TRUE) {
         header('HTTP/1.1 304 Not Modified');
@@ -373,14 +434,15 @@ class Server {
    * Returns server info
    */
   public function getInfo() {
+//    echo $this->config['baseUrls'][0];die;
     $this->setDatasets();
     $maps = array_merge($this->fileLayer, $this->dbLayer);
     header('Content-Type: text/html;charset=UTF-8');
     echo '<!DOCTYPE html><html><head><meta charset="UTF-8"><title>' . $this->config['serverTitle'] . '</title></head><body>';
     echo '<h1>' . $this->config['serverTitle'] . '</h1>';
-    echo 'TileJSON service: <a href="' . $this->config['baseUrls'][0] . '/index.json">' . $this->config['baseUrls'][0] . '/index.json</a><br>';
-    echo 'WMTS service: <a href="' . $this->config['baseUrls'][0] . '/wmts">' . $this->config['baseUrls'][0] . '/wmts</a><br>';
-    echo 'TMS service: <a href="' . $this->config['baseUrls'][0] . '/tms">' . $this->config['baseUrls'][0] . '/tms</a>';
+    echo 'TileJSON service: <a href="//' . $this->config['baseUrls'][0] . '/index.json">' . $this->config['baseUrls'][0] . '/index.json</a><br>';
+    echo 'WMTS service: <a href="//' . $this->config['baseUrls'][0] . '/wmts">' . $this->config['baseUrls'][0] . '/wmts</a><br>';
+    echo 'TMS service: <a href="//' . $this->config['baseUrls'][0] . '/tms">' . $this->config['baseUrls'][0] . '/tms</a>';
     foreach ($maps as $map) {
       $extend = '[';
       foreach ($map['bounds'] as $ext) {
@@ -392,7 +454,7 @@ class Server {
       } else {
         echo '<p>Available file tileset: ' . $map['basename'] . '<br>';
       }
-      echo 'Metadata: <a href="' . $this->config['baseUrls'][0] . '/' . $map['basename'] . '.json">'
+      echo 'Metadata: <a href="//' . $this->config['baseUrls'][0] . '/' . $map['basename'] . '.json">'
       . $this->config['baseUrls'][0] . '/' . $map['basename'] . '.json</a><br>';
       echo 'Bounds: ' . $extend . '</p>';
     }
@@ -410,7 +472,7 @@ class Server {
     echo '<!DOCTYPE html><html><head><meta charset="UTF-8"><title>' . $this->config['serverTitle'] . '</title>';
     echo '<link rel="stylesheet" type="text/css" href="//tileserver.com/v1/index.css" />
           <script src="//tileserver.com/v1/index.js"></script><body>
-          <script>tileserver({index:"http://' . $this->config['baseUrls'][0] . '/index.json", tilejson:"http://' . $this->config['baseUrls'][0] . '/%n.json", tms:"http://' . $this->config['baseUrls'][0] . '/tms", wmts:"http://' . $this->config['baseUrls'][0] . '/wmts"});</script>
+          <script>tileserver({index:"' . $this->config['protocol'] . '://' . $this->config['baseUrls'][0] . '/index.json", tilejson:"' . $this->config['protocol'] . '://' . $this->config['baseUrls'][0] . '/%n.json", tms:"' . $this->config['protocol'] . '://' . $this->config['baseUrls'][0] . '/tms", wmts:"' . $this->config['protocol'] . '://' . $this->config['baseUrls'][0] . '/wmts"});</script>
           <h1>Welcome to ' . $this->config['serverTitle'] . '</h1>
           <p>This server distributes maps to desktop, web, and mobile applications.</p>
           <p>The mapping data are available as OpenGIS Web Map Tiling Service (OGC WMTS), OSGEO Tile Map Service (TMS), and popular XYZ urls described with TileJSON metadata.</p>';
@@ -490,19 +552,26 @@ class Json extends Server {
     $metadata['scheme'] = 'xyz';
     $tiles = array();
     foreach ($this->config['baseUrls'] as $url) {
-      $tiles[] = 'http://' . $url . '/' . $metadata['basename'] . '/{z}/{x}/{y}.' . $metadata['format'];
+      $tiles[] = '' . $this->config['protocol'] . '://' . $url . '/' . $metadata['basename'] . '/{z}/{x}/{y}.' . $metadata['format'];
     }
     $metadata['tiles'] = $tiles;
     if ($this->isDBLayer($metadata['basename'])) {
       $this->DBconnect($metadata['basename'] . '.mbtiles');
-      $res = $this->db->query('SELECT grid FROM grids LIMIT 1');
+      $res = $this->db->query('SELECT name FROM sqlite_master WHERE name="grids";');
       if ($res) {
         foreach ($this->config['baseUrls'] as $url) {
-          $grids[] = 'http://' . $url . '/' . $metadata['basename'] . '/{z}/{x}/{y}.grid.json';
+          $grids[] = '' . $this->config['protocol'] . '://' . $url . '/' . $metadata['basename'] . '/{z}/{x}/{y}.grid.json';
         }
         $metadata['grids'] = $grids;
       }
     }
+    if (array_key_exists('json', $metadata)) {
+      $mjson = json_decode(stripslashes($metadata['json']));
+      foreach ($mjson as $key => $value) {
+        $metadata[$key] = $value;
+      }
+      unset($metadata['json']);
+    }
     return $metadata;
   }
 
@@ -535,7 +604,7 @@ class Json extends Server {
       echo 'TileServer: unknown map ' . $basename;
       die;
     }
-    return $output;
+    return stripslashes($output);
   }
 
   /**
@@ -544,7 +613,7 @@ class Json extends Server {
   public function getJson() {
     parent::setDatasets();
     header('Access-Control-Allow-Origin: *');
-    header("Content-Type:application/javascript charset=utf-8");
+    header("Content-Type: application/json; charset=utf-8");
     if ($this->callback !== 'grid') {
       echo $this->callback . '(' . $this->createJson($this->layer) . ');'; die;
     } else {
@@ -558,7 +627,7 @@ class Json extends Server {
   public function getJsonp() {
     parent::setDatasets();
     header('Access-Control-Allow-Origin: *');
-    header("Content-Type:text/javascript charset=utf-8");
+    header("Content-Type: application/javascript; charset=utf-8");
     echo $this->callback . '(' . $this->createJson($this->layer) . ');';
   }
 
@@ -566,7 +635,7 @@ class Json extends Server {
    * Returns UTFGrid in JSON format
    */
   public function getUTFGrid() {
-    parent::getUTFGrid($this->layer, $this->z, $this->y, $this->x);
+    parent::renderUTFGrid($this->layer, $this->z, $this->y, $this->x);
   }
 
 }
@@ -643,7 +712,7 @@ class Wmts extends Server {
     <ows:Operation name="GetCapabilities">
       <ows:DCP>
         <ows:HTTP>
-          <ows:Get xlink:href="http://' . $this->config['baseUrls'][0] . '/wmts/1.0.0/WMTSCapabilities.xml">
+          <ows:Get xlink:href="' . $this->config['protocol'] . '://' . $this->config['baseUrls'][0] . '/wmts/1.0.0/WMTSCapabilities.xml">
             <ows:Constraint name="GetEncoding">
               <ows:AllowedValues>
                 <ows:Value>RESTful</ows:Value>
@@ -651,7 +720,7 @@ class Wmts extends Server {
             </ows:Constraint>
           </ows:Get>
           <!-- add KVP binding in 10.1 -->
-          <ows:Get xlink:href="http://' . $this->config['baseUrls'][0] . '/wmts?">
+          <ows:Get xlink:href="' . $this->config['protocol'] . '://' . $this->config['baseUrls'][0] . '/wmts?">
             <ows:Constraint name="GetEncoding">
               <ows:AllowedValues>
                 <ows:Value>KVP</ows:Value>
@@ -664,14 +733,14 @@ class Wmts extends Server {
     <ows:Operation name="GetTile">
       <ows:DCP>
         <ows:HTTP>
-          <ows:Get xlink:href="http://' . $this->config['baseUrls'][0] . '/wmts/">
+          <ows:Get xlink:href="' . $this->config['protocol'] . '://' . $this->config['baseUrls'][0] . '/wmts/">
             <ows:Constraint name="GetEncoding">
               <ows:AllowedValues>
                 <ows:Value>RESTful</ows:Value>
               </ows:AllowedValues>
             </ows:Constraint>
           </ows:Get>
-          <ows:Get xlink:href="http://' . $this->config['baseUrls'][0] . '/wmts?">
+          <ows:Get xlink:href="' . $this->config['protocol'] . '://' . $this->config['baseUrls'][0] . '/wmts?">
             <ows:Constraint name="GetEncoding">
               <ows:AllowedValues>
                 <ows:Value>KVP</ows:Value>
@@ -719,7 +788,7 @@ class Wmts extends Server {
       <TileMatrixSetLink>
         <TileMatrixSet>' . $tileMatrixSet . '</TileMatrixSet>
       </TileMatrixSetLink>
-      <ResourceURL format="' . $mime . '" resourceType="tile" template="http://'
+      <ResourceURL format="' . $mime . '" resourceType="tile" template="' . $this->config['protocol'] . '://'
       . $this->config['baseUrls'][0] . '/wmts/' . $basename . '/{TileMatrixSet}/{TileMatrix}/{TileCol}/{TileRow}.' . $format . '"/>
     </Layer>';
     }
@@ -1075,7 +1144,7 @@ class Wmts extends Server {
       </TileMatrix>
     </TileMatrixSet>
   </Contents>
-  <ServiceMetadataURL xlink:href="http://' . $this->config['baseUrls'][0] . '/wmts/1.0.0/WMTSCapabilities.xml"/>
+  <ServiceMetadataURL xlink:href="' . $this->config['protocol'] . '://' . $this->config['baseUrls'][0] . '/wmts/1.0.0/WMTSCapabilities.xml"/>
 </Capabilities>';
   }
 
@@ -1091,9 +1160,9 @@ class Wmts extends Server {
       } else {
         $format = $this->getGlobal('Format');
       }
-      parent::getTile($this->getGlobal('Layer'), $this->getGlobal('TileMatrix'), $this->getGlobal('TileRow'), $this->getGlobal('TileCol'), $format);
+      parent::renderTile($this->getGlobal('Layer'), $this->getGlobal('TileMatrix'), $this->getGlobal('TileRow'), $this->getGlobal('TileCol'), $format);
     } else {
-      parent::getTile($this->layer, $this->z, $this->y, $this->x, $this->ext);
+      parent::renderTile($this->layer, $this->z, $this->y, $this->x, $this->ext);
     }
   }
 
@@ -1156,7 +1225,7 @@ class Tms extends Server {
         $srs = "EPSG:3857";
         echo '<TileMap title="' . $title . '" srs="' . $srs
         . '" type="InvertedTMS" ' . 'profile="global-' . $profile
-        . '" href="http://' . $this->config['baseUrls'][0] . '/tms/' . $basename . '" />';
+        . '" href="' . $this->config['protocol'] . '://' . $this->config['baseUrls'][0] . '/tms/' . $basename . '" />';
       }
     }
     echo '</TileMaps></TileMapService>';
@@ -1194,7 +1263,7 @@ class Tms extends Server {
     }
     $mime = ($m['format'] == 'jpg') ? 'image/jpeg' : 'image/png';
     header("Content-type: application/xml");
-    echo '<TileMap version="1.0.0" tilemapservice="http://' . $this->config['baseUrls'][0] . '/' . $m['basename'] . '" type="InvertedTMS">
+    echo '<TileMap version="1.0.0" tilemapservice="' . $this->config['protocol'] . '://' . $this->config['baseUrls'][0] . '/' . $m['basename'] . '" type="InvertedTMS">
   <Title>' . htmlspecialchars($title) . '</Title>
   <Abstract>' . htmlspecialchars($description) . '</Abstract>
   <SRS>' . $srs . '</SRS>
@@ -1203,7 +1272,7 @@ class Tms extends Server {
   <TileFormat width="256" height="256" mime-type="' . $mime . '" extension="' . $m['format'] . '"/>
   <TileSets profile="global-' . $m['profile'] . '">';
     for ($zoom = $m['minzoom']; $zoom < $m['maxzoom'] + 1; $zoom++) {
-      echo '<TileSet href="http://' . $this->config['baseUrls'] [0] . '/' . $m['basename'] . '/' . $zoom . '" units-per-pixel="' . $initialResolution / pow(2, $zoom) . '" order="' . $zoom . '" />';
+      echo '<TileSet href="' . $this->config['protocol'] . '://' . $this->config['baseUrls'] [0] . '/' . $m['basename'] . '/' . $zoom . '" units-per-pixel="' . $initialResolution / pow(2, $zoom) . '" order="' . $zoom . '" />';
     }
     echo'</TileSets></TileMap>';
   }
@@ -1212,7 +1281,7 @@ class Tms extends Server {
    * Process getTile request
    */
   public function getTile() {
-    parent::getTile($this->layer, $this->z, $this->y, $this->x, $this->ext);
+    parent::renderTile($this->layer, $this->z, $this->y, $this->x, $this->ext);
   }
 
 }
@@ -1356,6 +1425,8 @@ class Router {
   public static function serve($routes) {
     $request_method = strtolower($_SERVER['REQUEST_METHOD']);
     $path_info = '/';
+	global $config;
+	$config['protocol'] = ( isset($_SERVER["HTTPS"]) or $_SERVER['SERVER_PORT'] == '443') ? "https" : "http";
     if (!empty($_SERVER['PATH_INFO'])) {
       $path_info = $_SERVER['PATH_INFO'];
     } else if (!empty($_SERVER['ORIG_PATH_INFO']) && $_SERVER['ORIG_PATH_INFO'] !== '/tileserver.php') {
@@ -1375,9 +1446,9 @@ class Router {
       $tokens = array(
           ':string' => '([a-zA-Z]+)',
           ':number' => '([0-9]+)',
-          ':alpha' => '([a-zA-Z0-9-_]+)'
+          ':alpha' => '([a-zA-Z0-9-_@]+)'
       );
-      global $config;
+      //global $config;
       foreach ($routes as $pattern => $handler_name) {
         $pattern = strtr($pattern, $tokens);
         if (preg_match('#/?' . $pattern . '/?$#', $path_info, $matches)) {