merged

Author: davidraleigh

.dockerignore

@@ -0,0 +1,7 @@
+./build
+./dist
+./epl.egg-info
+./index
+./venv
+*.tar.gz
+**/__pycache__

.gitignore

@@ -1,10 +1,10 @@
-
 .idea
 venv
 index
 *.tar.gz
-__pycache__/
+**/__pycache__
 dist
 epl.egg-info
 .ipynb_checkpoints/
 build/
+naip_visual_db-*

Dockerfile

@@ -0,0 +1,16 @@
+FROM python:slim
+
+WORKDIR /opt/src/naip-stac-grpc
+COPY ./ ./
+
+RUN pip install -r requirements.txt
+
+RUN python -mgrpc_tools.protoc -I=./protos --python_out=./ \
+    ./protos/epl/protobuf/geometry_operators.proto ./protos/epl/protobuf/stac.proto \
+    ./protos/epl/protobuf/stac_item_result.proto
+RUN python -mgrpc_tools.protoc -I=./protos --grpc_python_out=./ \
+    ./protos/epl/grpc/naip_stac.proto
+
+RUN python setup.py install
+
+CMD python service.py

NAIP_TEST.ipynb

@@ -303,7 +303,7 @@
     "Session = rasterio.Env(\n",
     "    AWS_REQUEST_PAYER='requester',\n",
     "    GDAL_DISABLE_READDIR_ON_OPEN='YES',\n",
-    "    AWS_PROFILE='${HOME}/.aws/credentials')\n",
+    "    AWS_PROFILE='${HOME}/.aws/naip-download.csv')\n",
     "\n",
     "assets = metadata_result.assets\n",
     "for asset_key in assets:\n",

README.md

@@ -1,6 +1,56 @@
 # STAC + NAIP + gRPC Metadata Service
 This is a first version of a [gRPC](https://grpc.io/) service and [protobuf](https://developers.google.com/protocol-buffers/) definition for serving [NAIP](https://registry.opendata.aws/naip/) metadata that tries to be [STAC](https://github.com/radiantearth/stac-spec) compliant.
 
+## TLDR
+Requirements:
+* `aws` [cli tool](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-install.html)
+* AWS s3 requester pays authorization in ~/.aws/credentials (`aws configure` to setup)
+* `ogr2ogr` with postgres extensions
+* docker + docker-compose
+* [virtualenv](https://virtualenv.pypa.io/en/latest/installation/)
+
+#### Commands
+There are two different ways to test the gRPC service. One is with a postgis docker container and the other is with a docker-compose db+service. Both require this initial set of commands.
+
+Initial commands will download a gig or more of shapefiles to your local machine and prepare a database:
+```bash
+git clone git@github.com:geo-grpc/naip-stac-grpc.git
+cd naip-stac-grpc
+virtualenv venv
+source venv/bin/activate
+./download_shapefiles.sh
+./naip_import_aws.sh
+```
+
+Test version 1; test with the local docker database:
+```bash
+pip3 install -r requirements.txt
+python3 setup.py install
+python3 service.py
+python3 test_client.py
+```
+
+Test version 2; test with docker-compose initialized with a pg_dump file:
+```bash
+# dump postgres table to file
+docker exec naip-metadata-postgis pg_dump -U user -Fc \
+  -t naip_visual testdb > ./naip_visual_db-$(date +%Y-%m-%d)
+docker stop naip-metadata-postgis
+docker-compose up --build -d
+# wait for postgres db to initialize. you could omit the `-d` and 
+# watch for the initialization completion and execute the rest of 
+# the command from another window
+sleep 15
+docker exec -i naip-stac-grpc_db_1 pg_restore -C --clean --no-acl --no-owner \
+  -U user -d testdb < ./naip_visual_db-$(date +%Y-%m-%d)
+pip3 install -r requirements.txt
+python3 test_client.py
+```
+
+
+
+
+
 ## STAC, Protocol Buffers, and gPRC
 
 #### Protobuf Defintions
@@ -26,7 +76,7 @@ This stac experiment imports a geometry proto definition used in another gRPC pr
 #### Requirements
 install requirements:
 ```bash
-pip3 install -r requirements
+pip3 install -r requirements.txt
 ```
 
 #### Protoc Compile Step (Optional)
@@ -57,7 +107,7 @@ GDAL + ogr2ogr + postgresql:
 brew install gdal2 --with-postgresql
 ```
 
-To collect all the data data from the AWS NAIP shapefiles you'll need to execute the included bash script, `naip_import_aws.sh`.
+To collect all the data data from the AWS NAIP shapefiles (a gig or more) you'll need to execute the included bash script, `naip_import_aws.sh`.
 
 ## Testing
 

docker-compose.yml

@@ -0,0 +1,32 @@
+version: '3'
+services:
+  naip-stac-grpc:
+    container_name: naip-stac-grpc-service-c
+    image: us.gcr.io/echoparklabs/naip-stac-grpc
+    build:
+      context: .
+      dockerfile: Dockerfile
+    depends_on:
+      - db
+    networks:
+      - naip
+    ports:
+      - "50051:50051"
+    environment:
+      - POSTGRES_HOST=db
+      - POSTGRES_PORT=5432
+
+  db:
+    container_name: naip-stac-grpc-db-c
+    image: mdillon/postgis
+    ports:
+      - "5432:5432"
+    networks:
+      - naip
+    environment:
+      - POSTGRES_PASSWORD=cabbage
+      - POSTGRES_USER=user
+      - POSTGRES_DB=testdb
+
+networks:
+  naip: {}

download_shapefiles.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+STATES=(al ar az ca co ct de fl ga ia id il in ks ky la ma md me mi mn mo ms mt nc nd ne nh nj nm nv ny oh ok or pa ri sc sd tn tx ut va vt wa wi wv wy)
+END_YEAR=$(date +'%Y')
+FAILING_SHAPEFILES=(naip_3_172_6_fl naip_3_17_2_8_sc naip_3_17_2_2_pa)
+
+for state in "${STATES[@]}"
+do
+   for year in $(seq 2011 $END_YEAR)
+   do
+      echo s3 cp s3://naip-visualization/$state/$year/60cm/index/ ./index/  --request-payer --recursive
+      # if the bucket doesn't exist these copies won't do anything at all
+      aws s3 cp s3://naip-visualization/$state/$year/60cm/index/ ./index/  --request-payer --recursive
+      echo s3 cp s3://naip-visualization/$state/$year/100cm/index/ ./index/  --request-payer --recursive
+      aws s3 cp s3://naip-visualization/$state/$year/100cm/index/ ./index/  --request-payer --recursive
+   done
+done

epl/asset.py

@@ -88,8 +88,3 @@ def extract_naip_s3_path(metadata_request: stac_pb2.MetadataRequest,
                 write_asset(s3_path_template, imagery_bands, asset_type, metadata_result.assets)
         else:
             write_asset(s3_path_template, imagery_bands, stac_pb2.GEOTIFF, metadata_result.assets)
-
-
-
-
-

epl/store.py

@@ -23,7 +23,7 @@
 from epl import parse, asset
 from datetime import datetime, timezone
 
-from sqlalchemy import Table, Column, Integer, String, MetaData, Date, Float, create_engine
+from sqlalchemy import Table, Column, Integer, String, MetaData, Date, Float
 from geoalchemy2 import Geometry
 from geoalchemy2.elements import WKTElement, WKBElement
 from sqlalchemy.sql import select, and_

epl/store_test.py

@@ -1,14 +1,20 @@
 import unittest
-from epl.protobuf import geometry_operators_pb2 as geometry
-from epl.protobuf import stac_pb2 as stac
-from epl import store, parse
-from google.protobuf.timestamp_pb2 import Timestamp
+import os
 
 from datetime import datetime, timezone
 
 from sqlalchemy import Table, Column, Integer, String, MetaData, Date, Float, create_engine
 from geoalchemy2 import Geometry
 from sqlalchemy.sql import select, and_
+from google.protobuf.timestamp_pb2 import Timestamp
+
+from epl.protobuf import geometry_operators_pb2 as geometry
+from epl.protobuf import stac_pb2 as stac
+from epl import store, parse
+
+
+POSTGRES_HOST = os.getenv('POSTGRES_HOST', 'localhost')
+POSTGRES_PORT = os.getenv('POSTGRES_PORT', 5432)
 
 metadata = MetaData()
 naip_visual = Table('naip_visual', metadata,
@@ -22,7 +28,8 @@
 
 class TestStore(unittest.TestCase):
     def setUp(self):
-        engine = create_engine('postgresql://user:cabbage@localhost:5432/testdb', echo=True)
+        engine = create_engine('postgresql://user:cabbage@{0}:{1}/testdb'.format(POSTGRES_HOST, POSTGRES_PORT),
+                               echo=True)
         self.postgres_access = store.PostgresStore(db_engine=engine)
 
     def test_simple_date_2(self):
@@ -39,7 +46,8 @@ def test_simple_date_2(self):
         offset = 0
         current_and = None
         if metadata_req:
-            current_and = and_(naip_visual.c.srcimgdate >= datetime.fromtimestamp(metadata_req.src_img_date.value.seconds, timezone.utc))
+            current_and = and_(naip_visual.c.srcimgdate >= datetime.fromtimestamp(
+                metadata_req.src_img_date.value.seconds, timezone.utc))
         s = select([naip_visual], current_and).limit(limit).offset(offset)
 
         conn = self.postgres_access.db_engine.connect()

naip_import_aws.sh

@@ -3,26 +3,17 @@ STATES=(al ar az ca co ct de fl ga ia id il in ks ky la ma md me mi mn mo ms mt
 END_YEAR=$(date +'%Y')
 FAILING_SHAPEFILES=(naip_3_172_6_fl naip_3_17_2_8_sc naip_3_17_2_2_pa)
 
-for state in "${STATES[@]}"
-do
-   for year in $(seq 2011 $END_YEAR)
-   do
-      echo s3://naip-visualization/$state/$year/60cm/index/
-      # if the bucket doesn't exist these copies won't do anything at all
-      aws s3 cp s3://naip-visualization/$state/$year/60cm/index/ ./index/  --request-payer --recursive
-      echo s3://naip-visualization/$state/$year/100cm/index/
-      aws s3 cp s3://naip-visualization/$state/$year/100cm/index/ ./index/  --request-payer --recursive
-   done
-done
-
 cd index
 
 # TODO contact ESRI bucket owner about broken prj files
 # Failures
 # naip_3_13_2_1_al
 # GEOGCS["GCS_North_American_1983",DATUM["D_North_American_1983",SPHEROID["GRS_1980",6378137.0,298.257222101]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]]
 docker rm -f naip-metadata-postgis
-docker run -d --env POSTGRES_PASSWORD=cabbage --env POSTGRES_USER=user --env POSTGRES_DB=testdb -p 5432:5432 --name=naip-metadata-postgis mdillon/postgis
+docker run -d --env POSTGRES_PASSWORD=cabbage \
+              --env POSTGRES_USER=user \
+              --env POSTGRES_DB=testdb \
+              -p 5432:5432 --name=naip-metadata-postgis mdillon/postgis
 echo starting db
 sleep 40
 
@@ -83,8 +74,4 @@ docker exec -it naip-metadata-postgis /bin/bash -c "psql -qtA -d testdb -U user
 echo altering dates from varchar to dates
 docker exec -it naip-metadata-postgis /bin/bash -c "psql -qtA -d testdb -U user -c \"ALTER TABLE naip_visual ALTER COLUMN verdate TYPE DATE USING to_date(verdate, 'YYYYMMDD'); ALTER TABLE naip_visual ALTER COLUMN srcimgdate TYPE DATE USING to_date(srcimgdate, 'YYYYMMDD');\""
 
-# dump postgres table to file
-echo dumping everything to file
-docker exec naip-metadata-postgis pg_dump -U user -F t -t naip_visual testdb | gzip > ./naip_visual_db-$(date +%Y-%m-%d).tar.gz
-
 echo finished all tasks!

requirements-jupyter.txt

@@ -0,0 +1,4 @@
+jupyter
+rasterio
+boto3
+matplotlib

requirements-test.txt

@@ -0,0 +1,2 @@
+pytest
+pytest-flake8

service.py

@@ -38,6 +38,9 @@
 GRPC_CHAIN = os.getenv('GRPC_CHAIN', None)
 GRPC_KEY = os.getenv('GRPC_KEY', None)
 
+POSTGRES_HOST = os.getenv('POSTGRES_HOST', 'localhost')
+POSTGRES_PORT = os.getenv('POSTGRES_PORT', 5432)
+
 MAX_MESSAGE_MB = os.getenv('MAX_MESSAGE_MB', 64)
 BYTES_PER_MB = 1024 * 1024
 # https://github.com/grpc/grpc/issues/7927
@@ -66,7 +69,7 @@ def serve():
     server = grpc.server(futures.ThreadPoolExecutor(max_workers=10), options=GRPC_CHANNEL_OPTIONS)
 
     # db connection
-    engine = create_engine('postgresql://user:cabbage@localhost:5432/testdb', echo=True)
+    engine = create_engine('postgresql://user:cabbage@{0}:{1}/testdb'.format(POSTGRES_HOST, POSTGRES_PORT), echo=True)
 
     # add metadata service
     naip_grpc.add_MetadataOperatorsServicer_to_server(MetadataServicer(engine), server)

setup.cfg

@@ -0,0 +1,9 @@
+[tool:pytest]
+flake8-max-line-length = 120
+
+# these two can be ignored as they are generated or provided by google.
+flake8-ignore =
+    epl/protobuf/stac_pb2.py ALL
+    epl/protobuf/stac_item_result_pb2.py ALL
+    epl/protobuf/geometry_operators_pb2.py ALL
+    epl/grpc/naip_stac_pb2_grpc.py ALL

test_client.py

@@ -12,12 +12,12 @@
 
 MB = 1024 * 1024
 GRPC_CHANNEL_OPTIONS = [('grpc.max_message_length', 64 * MB), ('grpc.max_receive_message_length', 64 * MB)]
-GRPC_SERVICE_PORT = os.getenv('GRPC_SERVICE_PORT', 50051)
-GRPC_SERVICE_HOST = os.getenv('GRPC_SERVICE_HOST', 'localhost')
-IMAGERY_SERVICE = "{0}:{1}".format(GRPC_SERVICE_HOST, GRPC_SERVICE_PORT)
+NAIP_SERVICE_PORT = os.getenv('NAIP_SERVICE_PORT', 50051)
+NAIP_SERVICE_HOST = os.getenv('NAIP_SERVICE_HOST', 'localhost')
+IMAGERY_SERVICE = "{0}:{1}".format(NAIP_SERVICE_HOST, NAIP_SERVICE_PORT)
 ip_reg = re.compile(r"[\d]{1,3}\.[\d]{1,3}\.[\d]{1,3}\.[\d]{1,3}")
 
-if GRPC_SERVICE_HOST == "localhost" or ip_reg.match(GRPC_SERVICE_HOST):
+if NAIP_SERVICE_HOST == "localhost" or ip_reg.match(NAIP_SERVICE_HOST):
     channel = grpc.insecure_channel(IMAGERY_SERVICE, options=GRPC_CHANNEL_OPTIONS)
 else:
     channel_credentials = grpc.ssl_channel_credentials()