parse.py

import os
import urllib2
import math
import base64
import json
from operator import attrgetter
from sys import maxint as MAX_INT
from multiprocessing.pool import ThreadPool as Pool


def get_length(point1, point2):
    return math.sqrt((point1.x - point2.x) ** 2 + (point1.y - point2.y) ** 2)


def get_angle(angle_point, end_point1, end_point2):
    side_angle_point1 = get_length(angle_point, end_point1)
    side_angle_point2 = get_length(angle_point, end_point2)
    side_point1_point2 = get_length(end_point1, end_point2)
    if side_angle_point1 == 0 or side_angle_point2 == 0:
        return 0
    value = (side_angle_point1 ** 2 + side_angle_point2 ** 2 - side_point1_point2 ** 2) /\
            (2 * side_angle_point1 * side_angle_point2)
    return math.acos(min(max(value, -1), 1))


def get_next_polygon_point(angle_point, first_end_point, points):
    next_point = angle_point
    next_angle = 0
    for point in points:
        current_angle = get_angle(angle_point, first_end_point, point)
        if (current_angle > next_angle or
            current_angle == next_angle and
            get_length(angle_point, point) > get_length(angle_point, next_point)):
            next_angle = current_angle
            next_point = point
    return next_point


def get_polar_polygon_from_points(points):
    top_point = max(points, key=attrgetter('lng'))
    polygon_points = [top_point]
    while True:
        angle_point = polygon_points[-1]
        first_end_point = polygon_points[-2] if len(polygon_points) > 1 else angle_point.clone(delta_lat=1)
        next_point = get_next_polygon_point(angle_point, first_end_point, points)
        if top_point != next_point:
            polygon_points.append(next_point)
        else:
            break
    return polygon_points


def polar_to_int(lat, lng, zoom):
    x = int(2 ** zoom * (180 + lng) / 360)
    d = min(max(math.sin(lat * math.pi / 180), -0.9999), 0.9999)
    y = int(2 ** zoom * (2 * math.pi - math.log((1 + d) / (1 - d))) / (4 * math.pi))
    return Tile(x, y, zoom)


def polar_to_int_polygon(polar_polygon, zoom):
    return [polar_to_int(point.lat, point.lng, zoom) for point in polar_polygon]


def get_int_polygon_rectangle(int_polygon):
    x_top_left = MAX_INT
    y_top_left = MAX_INT
    x_bottom_right = 0
    y_bottom_right = 0
    zoom = int_polygon[0].zoom
    for point in int_polygon:
        x_top_left = min(x_top_left, point.x)
        y_top_left = min(y_top_left, point.y)
        x_bottom_right = max(x_bottom_right, point.x)
        y_bottom_right = max(y_bottom_right, point.y)
    return Tile(x_top_left, y_top_left, zoom), Tile(x_bottom_right, y_bottom_right, zoom)


def check_point_in_int_polygon(tile, int_polygon, imprecision=0.1):
    for polygon_point in int_polygon:
        if polygon_point == tile:
            return True
    angle_sum = 0
    for index, tile1 in enumerate(int_polygon):
        tile2 = int_polygon[index + 1] if index + 1 != len(int_polygon) else int_polygon[0]
        angle_sum += get_angle(tile, tile1, tile2)
    return angle_sum >= 2 * math.pi - imprecision


def create_dirs(save_file_path):
    dir_name = os.path.dirname(save_file_path)
    if not os.path.exists(dir_name):
        os.makedirs(dir_name)


def download_tile(tile, url_template, save_file_path_template):
    url = tile.render(url_template)
    save_file_path = tile.render(save_file_path_template)
    create_dirs(save_file_path)
    with open(save_file_path, 'wb') as file:
        file.write(urllib2.urlopen(url).read())


def check_and_download_tile(tile, url_template, save_file_path_template, int_polygon):
    if not check_point_in_int_polygon(tile, int_polygon):
        return
    download_tile(tile, url_template, save_file_path_template)


def download_tiles_in_polar_polygon(polar_polygon, zooms, url_template, save_file_path_template, threads_count=10):
    for zoom in zooms:
        int_polygon = polar_to_int_polygon(polar_polygon, zoom)
        point_top_left, point_bottom_right = get_int_polygon_rectangle(int_polygon)

        threads_pull = Pool(threads_count)

        for x in xrange(point_top_left.x, point_bottom_right.x + 1):
            for y in xrange(point_top_left.y, point_bottom_right.y + 1):
                threads_pull.apply_async(check_and_download_tile,
                    [Tile(x, y, zoom), url_template, save_file_path_template, int_polygon])

        threads_pull.close()
        threads_pull.join()


def get_images_path_list(root_path, from_root_path='', filter=''):
    for path, dirs, files in os.walk(root_path):
        for file in files:
            file_path = os.path.relpath(os.path.join(path, file), from_root_path)
            if file_path[-len(filter):] == filter:
                yield file_path


def get_zoom_and_coord_from_path(path):
    zoom = os.path.basename(os.path.dirname(path))
    coord = os.path.basename(path).split('.')[0]
    return '%s_%s' % (zoom, coord)


def image_to_base64(path):
    with open(path, 'rb') as file:
        return base64.b64encode(file.read())


def save_images_path_list(file_list, save_file):
    metadata_file_map = dict([(get_zoom_and_coord_from_path(file), file) for file in file_list])
    with open(save_file, 'wb') as file:
        json.dump(metadata_file_map, file)


def save_images_base64_list(file_list, save_file):
    metadata_file_map = dict([(get_zoom_and_coord_from_path(file), image_to_base64(file)) for file in file_list])
    with open(save_file, 'wb') as file:
        json.dump(metadata_file_map, file)


class Point():

    def __init__(self, lat, lng):
        self.lat = lat
        self.lng = lng

    def __eq__(self, other):
        return self.lat == other.lat and self.lng == other.lng

    def __ne__(self, other):
        return not self.__eq__(other)

    def __hash__(self):
        return hash((self.lat, self.lng))

    def __repr__(self):
        return '{lat: %(lat)s, lng: %(lng)s}' % {'lat': self.lat, 'lng': self.lng}

    def clone(self, delta_lat=0, delta_lng=0):
        return Point(self.lat + delta_lat, self.lng + delta_lng)

    @property
    def x(self):
        return self.lng

    @property
    def y(self):
        return self.lat


class Tile():

    def __init__(self, x, y, zoom):
        self.x = x
        self.y = y
        self.zoom = zoom

    def __eq__(self, other):
        return self.x == other.x and self.y == other.y and self.zoom == other.zoom

    def __ne__(self, other):
        return not self.__eq__(other)

    def __hash__(self):
        return hash((self.x, self.y, self.zoom))

    def __repr__(self):
        return self.render('{x: %(x)s, y: %(y)s, zoom: %(zoom)s}')

    def render(self, template):
        return template % {'x': self.x, 'y': self.y, 'zoom': self.zoom}


if __name__ == '__main__':
    from fixtures import all_points
    url_template = 'http://mt0.googleapis.com/vt?src=apiv3&x=%(x)s&y=%(y)s&z=%(zoom)s'
    project_path = 'site'
    save_files_path = project_path + '/cache'
    save_file_path_template = save_files_path + '/%(zoom)s/%(x)s_%(y)s.png'
    zooms = xrange(15 + 1)
    points = [Point(lat, lng) for lat, lng in all_points]
    polar_polygon = get_polar_polygon_from_points(points)
    download_tiles_in_polar_polygon(polar_polygon, zooms, url_template, save_file_path_template)

    images_path_list = save_files_path + '/images.json'
    file_list = list(get_images_path_list(save_files_path, project_path, '.png'))
    save_images_path_list(file_list, images_path_list)

    images_base64_list = save_files_path + '/imagesBase64.json'
    file_list = list(get_images_path_list(save_files_path, '', '.png'))
    save_images_base64_list(file_list, images_base64_list)