Upgrading chaos app to python3 runtime
BUG=chromium:1032597
Change-Id: Iaee3f06cfdfc881a75f654dd789bf729e9cd37bc
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/platform/crostestutils/+/2399180
Reviewed-by: Dinesh Kumar Sunkara <dsunkara@google.com>
Reviewed-by: Aashutosh Kalyankar <aashutoshk@chromium.org>
Reviewed-by: Katherine Threlkeld <kathrelkeld@chromium.org>
Tested-by: Aashutosh Kalyankar <aashutoshk@chromium.org>
Auto-Submit: Aashutosh Kalyankar <aashutoshk@chromium.org>
Commit-Queue: Aashutosh Kalyankar <aashutoshk@chromium.org>
diff --git a/provingground/chaosap/app.yaml b/provingground/chaosap/app.yaml
index f63c8df..e743522 100644
--- a/provingground/chaosap/app.yaml
+++ b/provingground/chaosap/app.yaml
@@ -1,8 +1,5 @@
-runtime: python27
-api_version: 1
-threadsafe: true
+runtime: python38
handlers:
- url: /.*
- script: main.app
- secure: always
+ script: auto
diff --git a/provingground/chaosap/main.py b/provingground/chaosap/main.py
index b386d8a..f30408f 100644
--- a/provingground/chaosap/main.py
+++ b/provingground/chaosap/main.py
@@ -1,6 +1,4 @@
-#!flask/bin/python
-# -*- coding: utf-8 -*-
-# Copyright 2017 Google Inc.
+# Copyright 2020 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -14,19 +12,17 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-import ap_spec
-import logging
-import ndb_json
from flask import Flask, jsonify, abort, request, make_response
-from google.appengine.ext import ndb
+from google.cloud import ndb
+from marshmallow import Schema, fields
app = Flask(__name__)
+client = ndb.Client()
-# Chaos AP Device Methods
class ApDevice(ndb.Model):
- """Models an individual AP object with specs."""
- hostname = ndb.StringProperty(indexed=True)
+ """ Model class to create AP entities to write to Datastore """
+ hostname = ndb.StringProperty()
lock_status = ndb.BooleanProperty(indexed=True)
lock_status_updated = ndb.DateTimeProperty(auto_now=True)
locked_by = ndb.StringProperty()
@@ -34,9 +30,25 @@
lab_label = ndb.StringProperty()
ap_label = ndb.StringProperty()
+
+class DataSerializer(Schema):
+
+ class Meta:
+ # The serialized output has all the items defined in fields param
+ fields = (
+ "id", "hostname", "lock_status", "lock_status_updated",
+ "locked_by", "router_name", "lab_label", "ap_label"
+ )
+
+
+ndb_serialize = DataSerializer()
+multi_ndb_serialize = DataSerializer(many=True)
+
+
@app.route('/')
def greeting():
- return 'Welcome to Chaos AP Page.\n'
+ return "Welcome to Chaos AP Page"
+
@app.route('/devices/new', methods=['POST'])
def post():
@@ -50,24 +62,23 @@
@returns string if success, error 400 if fails.
"""
if not request.json:
- abort(400)
- hostname = request.json['hostname']
- new_device = ApDevice(
- id = hostname,
- hostname = hostname,
- lock_status = False)
- if 'router_name' in request.json and type(request.json['router_name']) != unicode:
- make_response(jsonify({'error': 'Router name not unicode.'}), 400)
- elif 'ap_label' in request.json and type(request.json['ap_label']) != unicode:
- make_response(jsonify({'error': 'AP label not unicode.'}), 400)
- elif 'lab_label' in request.json and type(request.json['lab_label']) != unicode:
- make_response(jsonify({'error': 'Lab label not unicode.'}), 400)
- else:
+ abort(400, "Not a JSON request")
+ required_keys = ('hostname', 'router_name', 'ap_label', 'lab_label')
+ for _key in required_keys:
+ if _key in request.json and type(request.json[_key]) != str:
+ return make_response(jsonify({'error': f'{_key} value not a str'}))
+
+ with client.context():
+ hostname = request.json['hostname']
+ new_device = ApDevice(
+ id=hostname,
+ hostname=hostname,
+ lock_status=False)
new_device.router_name = request.json['router_name']
new_device.ap_label = request.json['ap_label']
new_device.lab_label = request.json['lab_label']
- new_device.put()
- return jsonify(result=True, id=hostname)
+ new_device.put()
+ return jsonify(result=True, id=hostname)
@app.route('/devices/<hostname>', methods=['PUT', 'GET'])
@@ -80,24 +91,21 @@
<URL>/devices/<HOSTNAME>
"""
- if request.method == 'GET':
+ with client.context():
device_key = ndb.Key(ApDevice, hostname)
device = device_key.get()
- device_js = ndb_json.dumps(device, indent=0)
- return device_js
-
- if request.method == 'PUT':
- device_key = ndb.Key(ApDevice, hostname)
- device = device_key.get()
- if 'router_name' in request.json:
- device.router_name = request.json['router_name']
- if 'ap_label' in request.json:
- device.ap_label = request.json['ap_label']
- if 'lab_label' in request.json:
- device.lab_label = request.json['lab_label']
- device.put()
- return jsonify(result=True, id=hostname, ap_label=device.ap_label, \
- lab_label=device.lab_label, router_name=device.router_name)
+ if request.method == 'GET':
+ return jsonify(ndb_serialize.dump(device))
+ if request.method == 'PUT':
+ if 'router_name' in request.json:
+ device.router_name = request.json['router_name']
+ if 'ap_label' in request.json:
+ device.ap_label = request.json['ap_label']
+ if 'lab_label' in request.json:
+ device.lab_label = request.json['lab_label']
+ device.put()
+ return jsonify(result=True, id=hostname, ap_label=device.ap_label,
+ lab_label=device.lab_label, router_name=device.router_name)
@app.route('/devices/lock', methods=['PUT'])
@@ -112,27 +120,31 @@
@returns Hosts locked message if success, error 400 if fails.
"""
if not request.json:
- abort(400)
- if type(request.json['hostname']) is list:
- hostnames = request.json['hostname']
- device_keys = [ndb.Key(ApDevice, x) for x in hostnames]
- devices = ndb.get_multi(device_keys)
- for d in devices:
- d.lock_status = True
- ndb.put_multi(devices)
- return jsonify(result=True, id=hostnames)
- elif 'hostname' in request.json and type(request.json['hostname']) != unicode:
- return make_response(jsonify({'error': 'Hostname not unicode.'}), 400)
- elif 'locked_by' in request.json and type(request.json['locked_by']) != unicode:
- return make_response(jsonify({'error': 'Need locked_by name'}), 400)
- else:
+ abort(400, "Not a JSON request")
+ with client.context():
hostname = request.json['hostname']
- device_key = ndb.Key(ApDevice, hostname)
- device = device_key.get()
- device.lock_status = True
- device.locked_by = request.json['locked_by']
- device.put()
- return jsonify(result=True, id=hostname)
+ if type(hostname) is list:
+ hostnames = hostname
+ device_keys = [ndb.Key(ApDevice, x) for x in hostnames]
+ devices = ndb.get_multi(device_keys)
+ for d in devices:
+ d.lock_status = True
+ d.locked_by = request.json['locked_by']
+ ndb.put_multi(devices)
+ return jsonify(result=True, id=hostnames)
+ elif 'hostname' in request.json and type(request.json['hostname']) != str:
+ return make_response(jsonify({'error': 'Hostname not string'}), 400)
+ elif 'locked_by' in request.json and type(request.json['locked_by']) != str:
+ return make_response(jsonify({'error': 'locked_by name not string'}), 400)
+ else:
+ hostname = request.json['hostname']
+ device_key = ndb.Key(ApDevice, hostname)
+ device = device_key.get()
+ device.lock_status = True
+ device.locked_by = request.json['locked_by']
+ device.put()
+ return jsonify(result=True, id=hostname)
+
@app.route('/devices/unlock', methods=['PUT'])
def unlock_devices():
@@ -146,24 +158,28 @@
"""
if not request.json:
return make_response(jsonify({'error': 'Need json dict.'}), 400)
- elif 'hostname' in request.json and type(request.json['hostname']) != unicode:
- return make_response(jsonify({'error': 'Hostname not unicode.'}), 400)
- elif type(request.json['hostname']) is list:
- hostnames = request.json['hostname']
- device_keys = [ndb.Key(ApDevice, x) for x in hostnames]
- devices = ndb.get_multi(device_keys)
- for d in devices:
- d.lock_status = False
- ndb.put_multi(devices)
- return jsonify(result=True, id=hostnames)
- else:
+ with client.context():
hostname = request.json['hostname']
- device_key = ndb.Key(ApDevice, hostname)
- device = device_key.get()
- device.lock_status = False
- device.locked_by = None
- device.put()
- return jsonify(result=True, id=hostname)
+ if type(hostname) is list:
+ hostnames = hostname
+ device_keys = [ndb.Key(ApDevice, x) for x in hostnames]
+ devices = ndb.get_multi(device_keys)
+ for d in devices:
+ d.lock_status = False
+ d.locked_by = None
+ ndb.put_multi(devices)
+ return jsonify(result=True, id=hostnames)
+
+ elif 'hostname' in request.json and type(request.json['hostname']) != str:
+ return make_response(jsonify({'error': 'Hostname not string'}), 400)
+ else:
+ device_key = ndb.Key(ApDevice, hostname)
+ device = device_key.get()
+ device.lock_status = False
+ device.locked_by = None
+ device.put()
+ return jsonify(result=True, id=hostname)
+
@app.route('/devices/delete', methods=['PUT'])
def delete():
@@ -174,13 +190,15 @@
@returns Hosts deleted message if success.
"""
- hostname = request.json['hostname']
- device_key = ndb.Key(ApDevice, hostname)
- device_key.delete()
- return "%s deleted.\n" % hostname
+ with client.context():
+ hostname = request.json['hostname']
+ device_key = ndb.Key(ApDevice, hostname)
+ device_key.delete()
+ return "%s deleted.\n" % hostname
# Querying devices.
+
@app.route('/devices/', methods=['GET'])
def get_devices():
"""Get list of devices in datastore.
@@ -189,9 +207,10 @@
@returns list of devices
"""
- devices_query = ApDevice.query()
- devices_entity = ndb_json.dumps(devices_query, indent=0, sort_keys=True)
- return devices_entity
+ with client.context():
+ devices_list = ApDevice.query().fetch()
+ return jsonify(multi_ndb_serialize.dump(devices_list))
+
@app.route('/devices/<hostname>', methods=['GET'])
def show_device(hostname):
@@ -201,10 +220,11 @@
@returns python dict of device
"""
- device_key = ndb.Key(ApDevice, hostname)
- device = device_key.get()
- device_js = ndb_json.dumps(device, indent=0)
- return device_js
+ with client.context():
+ device_key = ndb.Key(ApDevice, hostname)
+ device = device_key.get()
+ return jsonify(ndb_serialize.dump(device))
+
@app.route('/unlocked_devices/', methods=['GET'])
def get_unlocked_devices():
@@ -214,9 +234,10 @@
@returns list of devices
"""
- devices_query = ApDevice.query(ApDevice.lock_status==False)
- devices_entity = ndb_json.dumps(devices_query, indent=0, sort_keys=True)
- return devices_entity
+ with client.context():
+ devices_list = ApDevice.query(ApDevice.lock_status == False).fetch()
+ return jsonify(multi_ndb_serialize.dump(devices_list))
+
@app.route('/devices/location', methods=['PUT'])
def filter_by_location():
@@ -229,16 +250,17 @@
"""
if not request.json:
return make_response(jsonify({'error': 'Need json dict.'}), 400)
+ with client.context():
+ devices_list = ApDevice.query(ApDevice.ap_label == request.json["ap_label"],
+ ApDevice.lab_label == request.json["lab_label"],
+ ApDevice.lock_status == False).fetch()
+ return jsonify(multi_ndb_serialize.dump(devices_list))
- devices_query = ApDevice.query(ApDevice.ap_label==request.json["ap_label"],
- ApDevice.lab_label==request.json["lab_label"],
- ApDevice.lock_status==False)
- devices_entity = ndb_json.dumps(devices_query, indent=0, sort_keys=True)
- return devices_entity
@app.errorhandler(404)
def not_found(error):
return make_response(jsonify({'error': 'Not found'}), 404)
+
if __name__ == '__main__':
app.run(host='127.0.0.1', port=8080, debug=True)
diff --git a/provingground/chaosap/ndb_json.py b/provingground/chaosap/ndb_json.py
deleted file mode 100644
index 0437eb4..0000000
--- a/provingground/chaosap/ndb_json.py
+++ /dev/null
@@ -1,166 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-"""
-JSON encoder/decoder adapted for use with Google App Engine NDB.
-Usage:
- import ndb_json
-
- # Serialize an ndb.Query into an array of JSON objects.
- query = models.MyModel.query()
- query_json = ndb_json.dumps(query)
-
- # Convert into a list of Python dictionaries.
- query_dicts = ndb_json.loads(query_json)
-
- # Serialize an ndb.Model instance into a JSON object.
- entity = query.get()
- entity_json = ndb_json.dumps(entity)
-
- # Convert into a Python dictionary.
- entity_dict = ndb_json.loads(entity_json)
-Dependencies:
- - dateutil: https://pypi.python.org/pypi/python-dateutil
-"""
-
-__author__ = 'Eric Higgins'
-__copyright__ = 'Copyright 2013, Eric Higgins'
-__version__ = '0.0.5'
-__email__ = 'erichiggins@gmail.com'
-__status__ = 'Development'
-
-
-import base64
-import datetime
-import json
-import re
-import time
-import types
-
-import dateutil.parser
-from google.appengine.ext import ndb
-
-
-def encode_model(obj):
- """Encode objects like ndb.Model which have a `.to_dict()` method."""
- obj_dict = obj.to_dict()
- for key, val in obj_dict.iteritems():
- if isinstance(val, types.StringType):
- try:
- unicode(val)
- except UnicodeDecodeError:
- # Encode binary strings (blobs) to base64.
- obj_dict[key] = base64.b64encode(val)
- return obj_dict
-
-
-def encode_generator(obj):
- """Encode generator-like objects, such as ndb.Query."""
- return list(obj)
-
-
-def encode_key(obj):
- """Get the Entity from the ndb.Key for further encoding."""
- # Note(eric): Potentially poor performance for Models w/ many KeyProperty properties.
- return obj.get_async()
- # Alternative 1: Convert into pairs.
- #return obj.pairs()
- # Alternative 2: Convert into URL-safe base64-encoded string.
- #return obj.urlsafe()
-
-
-def encode_future(obj):
- """Encode an ndb.Future instance."""
- return obj.get_result()
-
-
-def encode_datetime(obj):
- """Encode a datetime.datetime or datetime.date object as an ISO 8601 format string."""
- # Reformat the date slightly for better JS compatibility.
- # Offset-naive dates need 'Z' appended for JS.
- # datetime.date objects don't have or need tzinfo, so don't append 'Z'.
- zone = '' if getattr(obj, 'tzinfo', True) else 'Z'
- return obj.isoformat() + zone
-
-
-def encode_complex(obj):
- """Convert a complex number object into a list containing the real and imaginary values."""
- return [obj.real, obj.imag]
-
-
-def encode_basevalue(obj):
- """Retrieve the actual value from a ndb.model._BaseValue.
-
- This is a convenience function to assist with the following issue:
- https://code.google.com/p/appengine-ndb-experiment/issues/detail?id=208
- """
- return obj.b_val
-
-
-NDB_TYPE_ENCODING = {
- ndb.MetaModel: encode_model,
- ndb.Query: encode_generator,
- ndb.QueryIterator: encode_generator,
- ndb.Key: encode_key,
- ndb.Future: encode_future,
- datetime.date: encode_datetime,
- datetime.datetime: encode_datetime,
- time.struct_time: encode_generator,
- types.ComplexType: encode_complex,
- ndb.model._BaseValue: encode_basevalue,
-
-}
-
-
-class NdbEncoder(json.JSONEncoder):
- """Extend the JSON encoder to add support for NDB Models."""
-
- def default(self, obj):
- """Overriding the default JSONEncoder.default for NDB support."""
-
- obj_type = type(obj)
- # NDB Models return a repr to calls from type().
- if obj_type not in NDB_TYPE_ENCODING and hasattr(obj, '__metaclass__'):
- obj_type = obj.__metaclass__
- fn = NDB_TYPE_ENCODING.get(obj_type)
- if fn:
- return fn(obj)
-
- return json.JSONEncoder.default(self, obj)
-
-
-def dumps(ndb_model, **kwargs):
- """Custom json dumps using the custom encoder above."""
- return NdbEncoder(**kwargs).encode(ndb_model)
-
-
-def dump(ndb_model, fp, **kwargs):
- """Custom json dump using the custom encoder above."""
- for chunk in NdbEncoder(**kwargs).iterencode(ndb_model):
- fp.write(chunk)
-
-
-def loads(json_str, **kwargs):
- """Custom json loads function that converts datetime strings."""
- json_dict = json.loads(json_str, **kwargs)
- if isinstance(json_dict, list):
- return map(iteritems, json_dict)
- return iteritems(json_dict)
-
-
-def iteritems(json_dict):
- """Loop over a json dict and try to convert strings to datetime."""
- for key, val in json_dict.iteritems():
- if isinstance(val, dict):
- iteritems(val)
- # Its a little hacky to check for specific chars, but avoids integers.
- elif isinstance(val, basestring) and 'T' in val:
- try:
- json_dict[key] = dateutil.parser.parse(val)
- # Check for UTC.
- if val.endswith(('+00:00', '-00:00', 'Z')):
- # Then remove tzinfo for gae, which is offset-naive.
- json_dict[key] = json_dict[key].replace(tzinfo=None)
- except (TypeError, ValueError):
- pass
- return json_dict
diff --git a/provingground/chaosap/requirements.txt b/provingground/chaosap/requirements.txt
index 0755008..c8fcfcb 100644
--- a/provingground/chaosap/requirements.txt
+++ b/provingground/chaosap/requirements.txt
@@ -1,6 +1,4 @@
-Flask==0.12.2
-oauth2client==4.1.2
-Flask-RESTful==0.3.6
-Flask-HTTPAuth==3.2.3
-PassLib==1.7.1
-python-dateutil==2.7.3
+Flask==1.1.2
+firebase-admin==4.3.0
+google-cloud-ndb==1.5.1
+marshmallow==3.7.1
