Created
November 27, 2015 13:14
-
-
Save rischanlab/3f96ffdd305531a4d80c to your computer and use it in GitHub Desktop.
script
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| __author__ = 'rischan' | |
| from django.core.exceptions import ObjectDoesNotExist | |
| from django.core.management.base import BaseCommand | |
| from agri_mapper.utilities.script import CallWMS, CallWFS, UpdateModel | |
| import logging | |
| logger = logging.getLogger(__name__) | |
| class Command(BaseCommand): | |
| help = 'This script is for synchronizing available layer data' | |
| def handle(self, *args, **options): | |
| """ Do your work here """ | |
| wms1 = CallWMS() | |
| updatemodel = UpdateModel() | |
| projects = updatemodel.get_project_names() | |
| for project in projects: | |
| try: | |
| wms = wms1.get_wms(project) | |
| wms_lists = (list(wms.contents)) | |
| except Exception as e: | |
| print "Oops! That project name {} was not available. " \ | |
| " Try again...".format(project) | |
| continue | |
| farm = updatemodel.get_farm_by_project_name(project) | |
| for layer_name in wms_lists: | |
| layer_name_has_date = updatemodel.layer_name_contain_date( | |
| layer_name) | |
| if layer_name_has_date: | |
| layer_data = updatemodel.get_dates_by_layer_name( | |
| layer_name) | |
| layer_filter = updatemodel.get_layer_filter( | |
| layer_data['layer_name']) | |
| # get layer_filter by clean layer_name | |
| if layer_filter is None: | |
| print "The layer {} is not " \ | |
| "in layer_filter table".format(layer_name) | |
| continue | |
| if updatemodel.check_layer_in_available_layer( | |
| farm, layer_filter, | |
| layer_data['date'], | |
| layer_data['week_number']): | |
| print "The layer {} has been " \ | |
| "in the available layer table".format( | |
| layer_name) | |
| continue | |
| try: | |
| updatemodel.insert_available_layer( | |
| farm, | |
| layer_filter, | |
| layer_data['date'], | |
| layer_data['week_number']) | |
| print "Success insert available layer " \ | |
| "for layer {}".format(layer_name) | |
| except ObjectDoesNotExist as e: | |
| print "Error in insert layer {} with date {} " \ | |
| "by date information in layer name".format( | |
| layer_name, date['date']) | |
| continue | |
| else: | |
| layer_filter = updatemodel.get_layer_filter(layer_name) | |
| wfs_layer_data = updatemodel.get_dates_by_wfs( | |
| project, layer_name) | |
| if layer_filter is None: | |
| print "The layer {} is not in layer_filter table".format( | |
| layer_name) | |
| continue | |
| if wfs_layer_data is not None: | |
| layer_data = updatemodel.get_dates_by_wfs( | |
| project, layer_name) | |
| for date in layer_data: | |
| if updatemodel.check_wfs_layer_in_available_layer( | |
| layer_filter, | |
| date=date['date'], | |
| week_number=date['week_number']): | |
| print "The layer {} with date {} has been " \ | |
| "in the available layer table".format( | |
| layer_name, date['date']) | |
| continue | |
| try: | |
| updatemodel.insert_available_layer( | |
| farm, | |
| layer_filter, | |
| date['date'], | |
| date['week_number']) | |
| print "Success insert available layer " \ | |
| "for layer {} with date {} ".format( | |
| layer_name, date['date']) | |
| except ObjectDoesNotExist as e: | |
| print "Error in insert layer {} with date {} " \ | |
| "by wfs".format(layer_name, date['date']) | |
| continue | |
| else: | |
| layer_data = updatemodel.get_date_from_current_date( | |
| layer_name) | |
| if updatemodel.check_layer_in_available_layer( | |
| farm, | |
| layer_filter, | |
| layer_data['date'], | |
| layer_data['week_number']): | |
| print "The layer {} has been " \ | |
| "in the available layer table".format(layer_name) | |
| continue | |
| try: | |
| updatemodel.insert_available_layer( | |
| farm, | |
| layer_filter, | |
| layer_data['date'], | |
| layer_data['week_number']) | |
| print "Success insert available layer " \ | |
| "for layer {} with date {} ".format( | |
| layer_name, layer_data['date']) | |
| except ObjectDoesNotExist as e: | |
| print "Error in insert layer {} with date {} " \ | |
| "by current_date".format( | |
| layer_name, layer_data['date']) | |
| continue |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| __author__ = 'rischan' | |
| from django.core.management.base import BaseCommand | |
| from agri_mapper.utilities.script import CallWMS, CallWFS, UpdateModel | |
| import logging | |
| logger = logging.getLogger(__name__) | |
| class Command(BaseCommand): | |
| help = 'This script is for synchronizing available layer data' | |
| def handle(self, *args, **options): | |
| """ Do your work here """ | |
| wms1 = CallWMS() | |
| updatemodel = UpdateModel() | |
| projects = updatemodel.get_project_names() | |
| for project in projects: | |
| try: | |
| wms = wms1.get_wms(project) | |
| wms_lists = (list(wms.contents)) | |
| farm = updatemodel.get_farm_by_project_name(project) | |
| for layer_name in wms_lists: | |
| layer_name_has_date = updatemodel.layer_name_contain_date(layer_name) | |
| if layer_name_has_date: | |
| layer_data = updatemodel.get_dates_by_layer_name(layer_name) | |
| layer_filter = updatemodel.get_layer_filter(layer_data['layer_name']) | |
| # get layer_filter by clean layer_name | |
| if layer_filter is not None: | |
| try: | |
| if not updatemodel.check_layer_in_available_layer(farm, layer_filter, layer_data['date'], layer_data['week_number']): | |
| updatemodel.insert_available_layer(farm, layer_filter, layer_data['date'], layer_data['week_number']) | |
| print "Success insert available layer for layer {}".format(layer_name) | |
| else: | |
| print "The layer {} has been in the available layer table".format(layer_name) | |
| except Exception as e: | |
| print "Error in insert layer {} with date {} by date information in layer name".format(layer_name, date['date']) | |
| continue | |
| else: | |
| print "The layer {} is not in layer_filter table".format(layer_name) | |
| else: | |
| layer_filter = updatemodel.get_layer_filter(layer_name) | |
| wfs_layer_data = updatemodel.get_dates_by_wfs(project, layer_name) | |
| # in this block layer_name is clean (does not has any date information) | |
| if layer_filter is not None: | |
| if wfs_layer_data is not None: | |
| layer_data = updatemodel.get_dates_by_wfs(project, layer_name) | |
| try: | |
| for date in layer_data: | |
| if not updatemodel.check_wfs_layer_in_available_layer(layer_filter, date=date['date'], week_number=date['week_number']): | |
| updatemodel.insert_available_layer(farm, layer_filter, date['date'], date['week_number']) | |
| print "Success insert available layer for layer {} with date {} ".format(layer_name, date['date']) | |
| else: | |
| print "The layer {} with date {} has been in the available layer table".format(layer_name, date['date']) | |
| except Exception as e: | |
| print "Error in insert layer {} with date {} by wfs".format(layer_name, date['date']) | |
| continue | |
| else: | |
| layer_data = updatemodel.get_date_from_current_date(layer_name) | |
| try: | |
| if not updatemodel.check_layer_in_available_layer(farm, layer_filter, layer_data['date'], layer_data['week_number']): | |
| updatemodel.insert_available_layer(farm, layer_filter, layer_data['date'], layer_data['week_number']) | |
| print "Success insert available layer for layer {} with date {} ".format(layer_name, layer_data['date']) | |
| else: | |
| print "The layer {} with date {} has been in the available layer table".format(layer_name, layer_data['date']) | |
| except Exception as e: | |
| print "Error in insert layer {} with date {} by current_date".format(layer_name, layer_data['date']) | |
| continue | |
| else: | |
| print "The layer {} is not in layer_filter table".format(layer_name) | |
| # except ValueError: | |
| # # If the project name is not valid. | |
| except Exception as e: | |
| print "Oops! That project name {} was not available. Try again...".format(project) | |
| continue |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| __author__ = 'rischan' | |
| import urllib | |
| import datetime | |
| from owslib.wms import WebMapService | |
| from agri_mapper.models.farm import Farm | |
| from agri_mapper.models.layer_filter import LayerFilter | |
| from agri_mapper.models.available_layer import AvailableLayer | |
| from xml.etree import ElementTree as ET | |
| static_url = 'http://129.232.164.58:6003/cgi-bin/qgis_mapserv.fcgi?map=/qgis/docs/projects/' | |
| extension = '.qgs' | |
| wms_version = '1.1.1' | |
| wfs_version = '1.0.0' | |
| farm_extent_layer = 'Plase' | |
| ns = { | |
| 'gml': 'http://www.opengis.net/gml', | |
| 'qgs': 'http://www.qgis.org/gml', | |
| } | |
| class CallWFS(): | |
| def wfs_url(self, project, layer_name): | |
| service = 'WFS' | |
| request = 'GetFeature' | |
| propertyname = 'datum' | |
| url = static_url + project + extension | |
| path = '&SERVICE=' + service + '&REQUEST=' + request + '&typeName=' + layer_name + '&propertyName=' + propertyname + '' | |
| xml_url = url+path | |
| return xml_url | |
| def get_wfs_dates(self, project, layer_name): | |
| try: | |
| url = self.wfs_url(project, layer_name) | |
| dates = [] | |
| root = ET.parse(urllib.urlopen(url)).getroot() | |
| for featureMember in root.findall('gml:featureMember', ns): | |
| for itest_leaf in featureMember: | |
| dates.append(itest_leaf.find('qgs:datum', ns).text) | |
| date_list = [x for x in dates if x is not None] | |
| return date_list | |
| except Exception as e: | |
| return None | |
| class CallWMS(): | |
| def get_capabilities_url(self): | |
| data = { | |
| 'URL': static_url, | |
| 'extension': extension, | |
| 'version': wms_version | |
| } | |
| return data | |
| def get_wms(self, project_name): | |
| data = self.get_capabilities_url() | |
| wms_url = data['URL'] + project_name + data['extension'] | |
| wms = WebMapService(wms_url, version=data['version']) | |
| return wms | |
| class UpdateModel(CallWMS, CallWFS): | |
| def get_project_names(self): | |
| return [farm.project_name for farm in Farm.objects.all()] | |
| def get_week_number(self, date): | |
| return date.isocalendar()[1] | |
| def check_layer_in_available_layer(self, farm, layer_filter, date, week_number): | |
| layer_is_there = False | |
| try: | |
| layer = AvailableLayer.objects.get(farm=farm, layer_filter=layer_filter, date_added=date, week_number=week_number) | |
| layer_is_there = True | |
| except Exception as e: | |
| layer_is_there = False | |
| return layer_is_there | |
| def check_wfs_layer_in_available_layer(self, layer_filter, date, week_number): | |
| layer_is_there = False | |
| try: | |
| layer = AvailableLayer.objects.get(layer_filter=layer_filter, date_added=date, week_number=week_number) | |
| layer_is_there = True | |
| except Exception as e: | |
| layer_is_there = False | |
| return layer_is_there | |
| def insert_available_layer( | |
| self, farm, layer_filter, date_added, week_number): | |
| layer = AvailableLayer( | |
| farm=farm, layer_filter=layer_filter, | |
| date_added=date_added, week_number=week_number) | |
| layer.save() | |
| def get_farm_by_project_name(self, project_name): | |
| farm = Farm.objects.get(project_name=project_name) | |
| return farm | |
| def get_layer_filter(self, layer_name): | |
| layer_filter = None | |
| try: | |
| layer_filter = LayerFilter.objects.get(layer_name=layer_name) | |
| return layer_filter | |
| except Exception as e: | |
| return layer_filter | |
| def layer_name_contain_date(self, layer_name): | |
| l = layer_name.split("_") | |
| if l[len(l)-1].isdigit() and l[len(l)-2].isdigit(): | |
| return True | |
| else: | |
| return False | |
| def get_dates_by_layer_name(self, layer_name): | |
| #layer_name = "Valskodlingmot_Larwes_2015_43" | |
| l = layer_name.split("_") | |
| length = len(l) | |
| year = l[length-2] | |
| wn = l[length-1] | |
| d = year+"-W"+wn | |
| date = datetime.datetime.strptime(d + '-0', "%Y-W%W-%w") | |
| name_length = length - 2 | |
| if name_length >= 2: | |
| name = l[:len(l)-2] | |
| layer_name = " ".join(name) | |
| else: | |
| layer_name = l[0] | |
| data = { | |
| 'layer_name' : layer_name, | |
| 'date' : date, | |
| 'week_number' : wn | |
| } | |
| return data | |
| def get_dates_by_wfs(self, project, layer_name): | |
| date = self.get_wfs_dates(project, layer_name) | |
| if date: | |
| unique_date = list(set(date)) | |
| data = [] | |
| for d in unique_date: | |
| week_number = self.get_week_number(datetime.datetime.strptime(d, "%Y/%m/%d")) | |
| data.append({ | |
| 'layer_name' : layer_name, | |
| 'date' : datetime.datetime.strptime(d, "%Y/%m/%d"), | |
| 'week_number' : week_number | |
| }) | |
| return data | |
| else: | |
| return None | |
| def get_date_from_current_date(self, layer_name): | |
| date = datetime.date.today() | |
| week_number = self.get_week_number(date) | |
| data = { | |
| 'layer_name' : layer_name, | |
| 'date' : date, | |
| 'week_number' : week_number | |
| } | |
| return data | |
| def get_farm_id(self, project_name): | |
| farm_id = None | |
| farm = Farm.objects.get(project_name=project_name) | |
| farm_id = farm.id | |
| return farm_id | |
| def check_extent(self, wms): | |
| if farm_extent_layer in wms: | |
| return True | |
| else: | |
| return False | |
| def get_extent(self, project_name): | |
| wms = self.get_wms(project_name) | |
| bbox = wms[farm_extent_layer].boundingBoxWGS84 | |
| data = { | |
| 'minx': bbox[0], | |
| 'miny': bbox[1], | |
| 'maxx': bbox[2], | |
| 'maxy': bbox[3] | |
| } | |
| return data | |
| def check_farm_extent_in_farm(self, project_name): | |
| bbox_is_there = False | |
| try: | |
| layer = Farm.objects.get(project_name=project_name) | |
| bbox = (layer.bbox_minx is None) or ( | |
| layer.bbox_miny is None) or ( | |
| layer.bbox_maxx is None) or ( | |
| layer.bbox_maxy is None) | |
| if bbox: | |
| bbox_is_there = False | |
| else: | |
| bbox_is_there = True | |
| except Exception as e: | |
| bbox_is_there = False | |
| return bbox_is_there | |
| def update_farm_extent(self, project_name, minx, miny, maxx, maxy): | |
| farm_id = self.get_farm_id(project_name) | |
| farm = Farm.objects.get(pk=farm_id) | |
| farm.bbox_minx = minx | |
| farm.bbox_miny = miny | |
| farm.bbox_maxx = maxx | |
| farm.bbox_maxy = maxy | |
| farm.save() | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment