From ecfcc14eb4af7ce1c42a16d36b551bf74d429b35 Mon Sep 17 00:00:00 2001 From: Adam Mcgrath Date: Sun, 9 Aug 2015 18:49:26 +0100 Subject: [PATCH 01/10] Initial commit of using GCS for file storage --- requirements.txt | 1 + src/forms.py | 40 +++++++++++++++++++++++++--------------- src/models.py | 1 + src/tests/test_forms.py | 12 ++++++------ 4 files changed, 33 insertions(+), 21 deletions(-) diff --git a/requirements.txt b/requirements.txt index 63f9f4b..6849521 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,3 +13,4 @@ python-coveralls python-dateutil==2.1 wtforms==0.6.3 gaenv==0.1.8.post0 +GoogleAppEngineCloudStorageClient==1.9.22.1 diff --git a/src/forms.py b/src/forms.py index 9f03476..3a2ffbb 100644 --- a/src/forms.py +++ b/src/forms.py @@ -12,12 +12,13 @@ import json import re import posixpath +import uuid +import cloudstorage import webapp2 from webapp2_extras import auth -from google.appengine.api import files -from google.appengine.api import urlfetch -from google.appengine.ext import ndb +from google.appengine.api import files, images, urlfetch +from google.appengine.ext import blobstore, ndb from google.appengine.ext.db import BadKeyError from wtforms import fields, Form, validators, widgets @@ -26,6 +27,7 @@ _USERNAME_RE = re.compile(r'^[\w\d_]{3,16}$') +GCS_FOLDER = '/ffcapp.appspot.com/images/' def validate_username(form, field): @@ -94,24 +96,32 @@ def populate_obj(self, obj, name): class ImageField(fields.FileField): """An image field.""" + def __init__(self, name, gcs_folder, img_size=None, **kwargs): + self.gcs_folder = gcs_folder + self.img_size = img_size + super(ImageField, self).__init__(name, **kwargs) + def populate_obj(self, obj, name): - """Populate the object represented by the film field.""" + """Populate the question model with a GCS image.""" req = webapp2.get_request() - img_file = req.get(self.name) - if not img_file: + if not req.get(self.name): return - file_name = files.blobstore.create(mime_type='application/octet-stream') - with files.open(file_name, 'a') as f: - f.write(img_file) - files.finalize(file_name) - setattr(obj, name, files.blobstore.get_blob_key(file_name)) + img_file = req.POST.get(self.name) + file_name = posixpath.join( + GCS_FOLDER, self.gcs_folder, uuid.uuid4(), img_file.filename) + gcs_file = cloudstorage.open(file_name, 'w', content_type=img_file.type) + logging.info('Saving file: %s' % file_name) + gcs_file.write(img_file.value) + gcs_file.close() + + setattr(obj, name, blobstore.create_gs_key('/gs' + file_name)) class ClueForm(Form): """A clue form.""" text = fields.TextAreaField('Text') - image = ImageField('Image') + image = ImageField('Image', 'clues') class ClueFormField(fields.FormField): @@ -208,7 +218,7 @@ def __init__(self, **kwargs): answer = FilmField('Film', [validators.Required()], id='film') clues = CluesFieldList(ClueFormField(ClueForm), min_entries=4) email_msg = fields.TextAreaField('Email Message') - packshot = ImageField('Image') + packshot = ImageField('Image', 'questions') imdb_url = fields.TextField('IMDB Link', default='http://www.imdb.com/title/XXX/') week = WeekField(choices=WeekField.week_choices()) @@ -223,14 +233,14 @@ class Registration(Form): class User(Form): username = fields.TextField('', [validate_username]) email = fields.TextField(validators=[validators.Email()]) - pic = ImageField('pic') + pic = ImageField('pic', 'profiles') favourite_film = FilmField() class League(Form): id = fields.HiddenField('') name = fields.TextField('', [validate_league_name]) - pic = ImageField('pic') + pic = ImageField('pic', 'leagues') owner = CurrentUserField() users = LeagueUsersField() diff --git a/src/models.py b/src/models.py index 5ff4954..5f3cdf4 100755 --- a/src/models.py +++ b/src/models.py @@ -66,6 +66,7 @@ class Question(ndb.Model): is_current = ndb.BooleanProperty(default=False) imdb_url = ndb.StringProperty() packshot = ndb.BlobKeyProperty() + packshot_img_url = ndb.StringProperty() email_msg = ndb.TextProperty() season = ndb.KeyProperty(kind=Season) week = ndb.IntegerProperty() diff --git a/src/tests/test_forms.py b/src/tests/test_forms.py index be9a416..f580861 100644 --- a/src/tests/test_forms.py +++ b/src/tests/test_forms.py @@ -137,12 +137,12 @@ def testFilmFieldPopulate(self): self.assertEqual(obj.foo_year, 'bar') self.assertEqual(obj.foo_title, 'baz') - def testImageFieldPopulate(self): - field = forms.ImageField().bind(Form(), 'a') - self.req.GET['foo'] = 'bar' - obj = mock.MagicMock() - field.populate_obj(obj, 'baz') - self.assertIsNotNone(obj.baz) + # def testImageFieldPopulate(self): + # field = forms.ImageField().bind(Form(), 'a') + # self.req.GET['foo'] = 'bar' + # obj = mock.MagicMock() + # field.populate_obj(obj, 'baz') + # self.assertIsNotNone(obj.baz) def testClueFieldListProcessNoClues(self): field_list = forms.CluesFieldList(FormField()).bind(Form(), 'a') From f62151cca031ab2141da0b1d82647d778b99c7ad Mon Sep 17 00:00:00 2001 From: Adam Mcgrath Date: Wed, 12 Aug 2015 21:51:15 +0100 Subject: [PATCH 02/10] Remove files from models --- src/auth.py | 7 ++++--- src/forms.py | 2 +- src/models.py | 23 +++++++++++++---------- src/tests/test_forms.py | 1 - src/views.py | 2 +- 5 files changed, 19 insertions(+), 16 deletions(-) diff --git a/src/auth.py b/src/auth.py index 8c30803..a59efb8 100644 --- a/src/auth.py +++ b/src/auth.py @@ -116,7 +116,8 @@ def _login_user(self, data, auth_info, provider): # Authenticate the new user. user.auth_ids.append(auth_id) - user.populate(**self._to_user_model_attrs(data, provider, True)) + user.populate(**self._to_user_model_attrs( + data, provider, True, user.username_lower)) user.put() self.auth.set_session(self.auth.store.user_to_dict(user), remember=True) del self.session['username'] @@ -165,7 +166,7 @@ def _get_consumer_info_for(self, provider): return secrets.get_auth_config( provider, settings.get_environment(self.request.host)) - def _to_user_model_attrs(self, data, provider, new_user): + def _to_user_model_attrs(self, data, provider, new_user, username): attrs_map = self.USER_ATTRS[provider] user_attrs = {} for k, v in data.iteritems(): @@ -182,7 +183,7 @@ def _to_user_model_attrs(self, data, provider, new_user): if new_user: if key == 'pic': - value = models.User.blob_from_url(value) + value = models.User.blob_from_url(value, username) user_attrs.setdefault(key, value) return user_attrs diff --git a/src/forms.py b/src/forms.py index 3a2ffbb..707ca13 100644 --- a/src/forms.py +++ b/src/forms.py @@ -17,7 +17,7 @@ import webapp2 from webapp2_extras import auth -from google.appengine.api import files, images, urlfetch +from google.appengine.api import images, urlfetch from google.appengine.ext import blobstore, ndb from google.appengine.ext.db import BadKeyError from wtforms import fields, Form, validators, widgets diff --git a/src/models.py b/src/models.py index 5f3cdf4..82a0c4f 100755 --- a/src/models.py +++ b/src/models.py @@ -9,15 +9,15 @@ import datetime import json import logging -import hashlib +import posixpath import re -import time import uuid +import cloudstorage from webapp2_extras.appengine.auth.models import User as AuthUser -from google.appengine.api import files, images, urlfetch -from google.appengine.ext import ndb +from google.appengine.api import images, urlfetch +from google.appengine.ext import blobstore, ndb from api import leaderboard import settings @@ -237,14 +237,17 @@ def get_by_username(username): return User.query().filter(User.username_lower == username.lower()).get() @staticmethod - def blob_from_url(url): + def blob_from_url(url, username): result = urlfetch.fetch(url) if result.status_code == 200: - file_name = files.blobstore.create(mime_type='application/octet-stream') - with files.open(file_name, 'a') as f: - f.write(result.content) - files.finalize(file_name) - return files.blobstore.get_blob_key(file_name) + file_name = posixpath.join( + '/ffcapp.appspot.com/images/profiles/', username) + gcs_file = cloudstorage.open( + file_name, 'w', content_type=result.headers['Content-Type']) + logging.info('Saving file: %s' % file_name) + gcs_file.write(result.content) + gcs_file.close() + return blobstore.create_gs_key('/gs' + file_name) @staticmethod def to_leaderboard_json(user): diff --git a/src/tests/test_forms.py b/src/tests/test_forms.py index f580861..126d50b 100644 --- a/src/tests/test_forms.py +++ b/src/tests/test_forms.py @@ -33,7 +33,6 @@ def setUp(self): self.req = req self.testbed.init_datastore_v3_stub() self.testbed.init_blobstore_stub() - self.testbed.init_files_stub() def testValidateValidUsername(self): field = mock.MagicMock() diff --git a/src/views.py b/src/views.py index 3a8161f..d09b6e9 100755 --- a/src/views.py +++ b/src/views.py @@ -10,7 +10,7 @@ from operator import itemgetter import uuid -from google.appengine.api import channel, files, users +from google.appengine.api import channel, users from google.appengine.ext import ndb import auth From 907a104acc8ffcc7aef49012dcc539ed8cb780e7 Mon Sep 17 00:00:00 2001 From: Adam Mcgrath Date: Sun, 9 Aug 2015 18:49:26 +0100 Subject: [PATCH 03/10] Initial commit of using GCS for file storage --- requirements.txt | 1 + src/forms.py | 40 +++++++++++++++++++++++++--------------- src/models.py | 1 + src/tests/test_forms.py | 12 ++++++------ 4 files changed, 33 insertions(+), 21 deletions(-) diff --git a/requirements.txt b/requirements.txt index 63f9f4b..6849521 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,3 +13,4 @@ python-coveralls python-dateutil==2.1 wtforms==0.6.3 gaenv==0.1.8.post0 +GoogleAppEngineCloudStorageClient==1.9.22.1 diff --git a/src/forms.py b/src/forms.py index 9f03476..3a2ffbb 100644 --- a/src/forms.py +++ b/src/forms.py @@ -12,12 +12,13 @@ import json import re import posixpath +import uuid +import cloudstorage import webapp2 from webapp2_extras import auth -from google.appengine.api import files -from google.appengine.api import urlfetch -from google.appengine.ext import ndb +from google.appengine.api import files, images, urlfetch +from google.appengine.ext import blobstore, ndb from google.appengine.ext.db import BadKeyError from wtforms import fields, Form, validators, widgets @@ -26,6 +27,7 @@ _USERNAME_RE = re.compile(r'^[\w\d_]{3,16}$') +GCS_FOLDER = '/ffcapp.appspot.com/images/' def validate_username(form, field): @@ -94,24 +96,32 @@ def populate_obj(self, obj, name): class ImageField(fields.FileField): """An image field.""" + def __init__(self, name, gcs_folder, img_size=None, **kwargs): + self.gcs_folder = gcs_folder + self.img_size = img_size + super(ImageField, self).__init__(name, **kwargs) + def populate_obj(self, obj, name): - """Populate the object represented by the film field.""" + """Populate the question model with a GCS image.""" req = webapp2.get_request() - img_file = req.get(self.name) - if not img_file: + if not req.get(self.name): return - file_name = files.blobstore.create(mime_type='application/octet-stream') - with files.open(file_name, 'a') as f: - f.write(img_file) - files.finalize(file_name) - setattr(obj, name, files.blobstore.get_blob_key(file_name)) + img_file = req.POST.get(self.name) + file_name = posixpath.join( + GCS_FOLDER, self.gcs_folder, uuid.uuid4(), img_file.filename) + gcs_file = cloudstorage.open(file_name, 'w', content_type=img_file.type) + logging.info('Saving file: %s' % file_name) + gcs_file.write(img_file.value) + gcs_file.close() + + setattr(obj, name, blobstore.create_gs_key('/gs' + file_name)) class ClueForm(Form): """A clue form.""" text = fields.TextAreaField('Text') - image = ImageField('Image') + image = ImageField('Image', 'clues') class ClueFormField(fields.FormField): @@ -208,7 +218,7 @@ def __init__(self, **kwargs): answer = FilmField('Film', [validators.Required()], id='film') clues = CluesFieldList(ClueFormField(ClueForm), min_entries=4) email_msg = fields.TextAreaField('Email Message') - packshot = ImageField('Image') + packshot = ImageField('Image', 'questions') imdb_url = fields.TextField('IMDB Link', default='http://www.imdb.com/title/XXX/') week = WeekField(choices=WeekField.week_choices()) @@ -223,14 +233,14 @@ class Registration(Form): class User(Form): username = fields.TextField('', [validate_username]) email = fields.TextField(validators=[validators.Email()]) - pic = ImageField('pic') + pic = ImageField('pic', 'profiles') favourite_film = FilmField() class League(Form): id = fields.HiddenField('') name = fields.TextField('', [validate_league_name]) - pic = ImageField('pic') + pic = ImageField('pic', 'leagues') owner = CurrentUserField() users = LeagueUsersField() diff --git a/src/models.py b/src/models.py index 5ff4954..5f3cdf4 100755 --- a/src/models.py +++ b/src/models.py @@ -66,6 +66,7 @@ class Question(ndb.Model): is_current = ndb.BooleanProperty(default=False) imdb_url = ndb.StringProperty() packshot = ndb.BlobKeyProperty() + packshot_img_url = ndb.StringProperty() email_msg = ndb.TextProperty() season = ndb.KeyProperty(kind=Season) week = ndb.IntegerProperty() diff --git a/src/tests/test_forms.py b/src/tests/test_forms.py index be9a416..f580861 100644 --- a/src/tests/test_forms.py +++ b/src/tests/test_forms.py @@ -137,12 +137,12 @@ def testFilmFieldPopulate(self): self.assertEqual(obj.foo_year, 'bar') self.assertEqual(obj.foo_title, 'baz') - def testImageFieldPopulate(self): - field = forms.ImageField().bind(Form(), 'a') - self.req.GET['foo'] = 'bar' - obj = mock.MagicMock() - field.populate_obj(obj, 'baz') - self.assertIsNotNone(obj.baz) + # def testImageFieldPopulate(self): + # field = forms.ImageField().bind(Form(), 'a') + # self.req.GET['foo'] = 'bar' + # obj = mock.MagicMock() + # field.populate_obj(obj, 'baz') + # self.assertIsNotNone(obj.baz) def testClueFieldListProcessNoClues(self): field_list = forms.CluesFieldList(FormField()).bind(Form(), 'a') From 1349799490c46c707ad4758a64dd8962c543861c Mon Sep 17 00:00:00 2001 From: Adam Mcgrath Date: Wed, 12 Aug 2015 21:51:15 +0100 Subject: [PATCH 04/10] Remove files from models --- src/auth.py | 7 ++++--- src/forms.py | 2 +- src/models.py | 23 +++++++++++++---------- src/tests/test_forms.py | 1 - src/views.py | 2 +- 5 files changed, 19 insertions(+), 16 deletions(-) diff --git a/src/auth.py b/src/auth.py index 8c30803..a59efb8 100644 --- a/src/auth.py +++ b/src/auth.py @@ -116,7 +116,8 @@ def _login_user(self, data, auth_info, provider): # Authenticate the new user. user.auth_ids.append(auth_id) - user.populate(**self._to_user_model_attrs(data, provider, True)) + user.populate(**self._to_user_model_attrs( + data, provider, True, user.username_lower)) user.put() self.auth.set_session(self.auth.store.user_to_dict(user), remember=True) del self.session['username'] @@ -165,7 +166,7 @@ def _get_consumer_info_for(self, provider): return secrets.get_auth_config( provider, settings.get_environment(self.request.host)) - def _to_user_model_attrs(self, data, provider, new_user): + def _to_user_model_attrs(self, data, provider, new_user, username): attrs_map = self.USER_ATTRS[provider] user_attrs = {} for k, v in data.iteritems(): @@ -182,7 +183,7 @@ def _to_user_model_attrs(self, data, provider, new_user): if new_user: if key == 'pic': - value = models.User.blob_from_url(value) + value = models.User.blob_from_url(value, username) user_attrs.setdefault(key, value) return user_attrs diff --git a/src/forms.py b/src/forms.py index 3a2ffbb..707ca13 100644 --- a/src/forms.py +++ b/src/forms.py @@ -17,7 +17,7 @@ import webapp2 from webapp2_extras import auth -from google.appengine.api import files, images, urlfetch +from google.appengine.api import images, urlfetch from google.appengine.ext import blobstore, ndb from google.appengine.ext.db import BadKeyError from wtforms import fields, Form, validators, widgets diff --git a/src/models.py b/src/models.py index 5f3cdf4..82a0c4f 100755 --- a/src/models.py +++ b/src/models.py @@ -9,15 +9,15 @@ import datetime import json import logging -import hashlib +import posixpath import re -import time import uuid +import cloudstorage from webapp2_extras.appengine.auth.models import User as AuthUser -from google.appengine.api import files, images, urlfetch -from google.appengine.ext import ndb +from google.appengine.api import images, urlfetch +from google.appengine.ext import blobstore, ndb from api import leaderboard import settings @@ -237,14 +237,17 @@ def get_by_username(username): return User.query().filter(User.username_lower == username.lower()).get() @staticmethod - def blob_from_url(url): + def blob_from_url(url, username): result = urlfetch.fetch(url) if result.status_code == 200: - file_name = files.blobstore.create(mime_type='application/octet-stream') - with files.open(file_name, 'a') as f: - f.write(result.content) - files.finalize(file_name) - return files.blobstore.get_blob_key(file_name) + file_name = posixpath.join( + '/ffcapp.appspot.com/images/profiles/', username) + gcs_file = cloudstorage.open( + file_name, 'w', content_type=result.headers['Content-Type']) + logging.info('Saving file: %s' % file_name) + gcs_file.write(result.content) + gcs_file.close() + return blobstore.create_gs_key('/gs' + file_name) @staticmethod def to_leaderboard_json(user): diff --git a/src/tests/test_forms.py b/src/tests/test_forms.py index f580861..126d50b 100644 --- a/src/tests/test_forms.py +++ b/src/tests/test_forms.py @@ -33,7 +33,6 @@ def setUp(self): self.req = req self.testbed.init_datastore_v3_stub() self.testbed.init_blobstore_stub() - self.testbed.init_files_stub() def testValidateValidUsername(self): field = mock.MagicMock() diff --git a/src/views.py b/src/views.py index 3a8161f..d09b6e9 100755 --- a/src/views.py +++ b/src/views.py @@ -10,7 +10,7 @@ from operator import itemgetter import uuid -from google.appengine.api import channel, files, users +from google.appengine.api import channel, users from google.appengine.ext import ndb import auth From a7ec86140f80441f03d9f8af006acf42f495b9d3 Mon Sep 17 00:00:00 2001 From: Adam Mcgrath Date: Sat, 22 Aug 2015 15:28:06 +0100 Subject: [PATCH 05/10] Fix auth tests --- src/auth.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/auth.py b/src/auth.py index a59efb8..e9160e2 100644 --- a/src/auth.py +++ b/src/auth.py @@ -90,7 +90,8 @@ def _login_user(self, data, auth_info, provider): if user: logging.info('Found existing user to log in') # existing user. just log them in and update token. - user.populate(**self._to_user_model_attrs(data, provider, False)) + user.populate(**self._to_user_model_attrs( + data, provider, False, user.username_lower)) user.put() self.auth.set_session(self.auth.store.user_to_dict(user), remember=True) @@ -104,7 +105,8 @@ def _login_user(self, data, auth_info, provider): logging.info('Updating currently logged in user.') user = self.current_user user.auth_ids.append(auth_id) - user.populate(**self._to_user_model_attrs(data, provider, False)) + user.populate(**self._to_user_model_attrs( + data, provider, False, user.username_lower)) user.put() self.auth.set_session(self.auth.store.user_to_dict(user), remember=True) From 2e1e8873ec6d662d44b7eafd825e90a88c25e34b Mon Sep 17 00:00:00 2001 From: Adam Mcgrath Date: Sun, 23 Aug 2015 13:40:22 +0100 Subject: [PATCH 06/10] Custom image path --- scripts/cloud_storage_migration.py | 101 +++++++++++++++++++++++++++++ src/forms.py | 101 +++++++++++++++++++++++++---- src/models.py | 2 +- 3 files changed, 192 insertions(+), 12 deletions(-) create mode 100755 scripts/cloud_storage_migration.py diff --git a/scripts/cloud_storage_migration.py b/scripts/cloud_storage_migration.py new file mode 100755 index 0000000..ddad3dc --- /dev/null +++ b/scripts/cloud_storage_migration.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python +# coding=utf-8 +# +# Copyright 2011 Friday Film Club. All Rights Reserved. + +"""Migrate images from blobstore to cloud storage + +You'll need to `pip install python-magic` +""" + +import os +import sys +sys.path.append('/usr/local/google_appengine') +import dev_appserver +import posixpath +import cloudstorage +import magic +import re +import getpass + +dev_appserver.fix_sys_path() + +dir = os.path.dirname(__file__) +sys.path.insert(0, os.path.join(dir, '../src')) + +from google.appengine.ext.remote_api import remote_api_stub +from google.appengine.ext import blobstore, ndb + +GCS_ROOT = '/ffcapp.appspot.com/images/' + +os.environ['SERVER_SOFTWARE'] = 'Development (remote_api)/1.0' + +import models + +APP_NAME = 's~ffcapp' +RE_SPECIAL_CHARS_ = re.compile(r'[^a-zA-Z0-9 ]') +os.environ['AUTH_DOMAIN'] = 'gmail.com' +os.environ['USER_EMAIL'] = 'adamjmcgrath@gmail.com' + +def slugify(my_string): + """Remove special characters and replace spaces with hyphens.""" + return '-'.join(re.sub(RE_SPECIAL_CHARS_, '', my_string).lower().split(' ')) + +def auth_func(): + return (os.environ['USER_EMAIL'], getpass.getpass()) + +def save_image(blob_key, folder_name, file_name): + img_file = blobstore.BlobInfo(blob_key) + img_file_o = img_file.open() + img_data = img_file_o.read() + type = magic.from_buffer(img_data, mime=True) + + file_path = posixpath.join( + GCS_ROOT, folder_name, file_name) + gcs_file = cloudstorage.open(file_path, 'w', content_type=type) + print 'Saving file: %s' % file_path + gcs_file.write(img_data) + gcs_file.close() + img_file_o.close() + return blobstore.BlobKey(blobstore.create_gs_key('/gs' + file_path)) + + +def migrate_screenshot(q): + clue = q.clues[0] + if not clue: + return + title = q.answer_title + clue_entity = clue.get() + old_key = str(clue_entity.image) + clue_entity.image = save_image( + clue_entity.image, 'questions', slugify(title) + '-screenshot') + clue_entity.put() + print ('screenshot: %s | old: %s | new: %s' % (title, old_key, str(clue_entity.image))) + + +def migrate_packshot(q): + title = q.answer_title + old_key = str(q.packshot) + q.packshot = save_image( + q.packshot, 'questions', slugify(title) + '-packshot') + q.put() + print ('packshot: %s | old: %s | new: %s' % (title, old_key, str(q.packshot))) + + +def migrate_questions(): + for q in models.Question.query().fetch(2): + migrate_packshot(q) + migrate_screenshot(q) + + +def main(): + # Use 'localhost:8080' for dev server. + remote_api_stub.ConfigureRemoteApi(APP_NAME, '/_ah/remote_api', + auth_func, servername='ffcapp.appspot.com') + + # migrate_questions() + # TODO leagues, users + + +if __name__ == '__main__': + main() diff --git a/src/forms.py b/src/forms.py index 707ca13..d235fbf 100644 --- a/src/forms.py +++ b/src/forms.py @@ -12,7 +12,6 @@ import json import re import posixpath -import uuid import cloudstorage import webapp2 @@ -27,7 +26,7 @@ _USERNAME_RE = re.compile(r'^[\w\d_]{3,16}$') -GCS_FOLDER = '/ffcapp.appspot.com/images/' +GCS_ROOT = '/ffcapp.appspot.com/images/' def validate_username(form, field): @@ -101,6 +100,10 @@ def __init__(self, name, gcs_folder, img_size=None, **kwargs): self.img_size = img_size super(ImageField, self).__init__(name, **kwargs) + def file_name(self, req, obj): + img_file = req.POST.get(self.name) + return img_file.filename + def populate_obj(self, obj, name): """Populate the question model with a GCS image.""" req = webapp2.get_request() @@ -109,19 +112,44 @@ def populate_obj(self, obj, name): img_file = req.POST.get(self.name) file_name = posixpath.join( - GCS_FOLDER, self.gcs_folder, uuid.uuid4(), img_file.filename) + GCS_ROOT, self.gcs_folder, self.file_name(req, obj)) gcs_file = cloudstorage.open(file_name, 'w', content_type=img_file.type) logging.info('Saving file: %s' % file_name) gcs_file.write(img_file.value) gcs_file.close() - setattr(obj, name, blobstore.create_gs_key('/gs' + file_name)) + setattr(obj, name, + blobstore.BlobKey(blobstore.create_gs_key('/gs' + file_name))) + + +class ProfileImageField(ImageField): + + def file_name(self, req, obj): + return obj.username_lower + + +class PackshotImageField(ImageField): + + def file_name(self, req, obj): + return '%s-packshot' % models.slugify(obj.answer_title) + + +class ScreenshotImageField(ImageField): + + def file_name(self, req, obj): + return '%s-screenshot' % models.slugify(obj.question.get().answer_title) + + +class LeagueImageField(ImageField): + + def file_name(self, req, obj): + return obj.name_slug class ClueForm(Form): """A clue form.""" text = fields.TextAreaField('Text') - image = ImageField('Image', 'clues') + image = ScreenshotImageField('Image', 'questions') class ClueFormField(fields.FormField): @@ -208,7 +236,41 @@ def populate_obj(self, entity, name): entity.users = [ndb.Key('User', int(key)) for key in self.data.split(',')] -class Question(Form): +class OrderedFieldForm(Form): + """ + Set the order in which the fields populate the model. + + So we can rely on the model having populated fields in subsequent + populate object calls. + """ + + # The field order list must contain the name of every field. + field_order = [] + + def populate_obj(self, obj): + """ + Populates the attributes of the passed `obj` with data from the form's + fields. + """ + items = sorted( + self._fields.items(), key=lambda tup: self.field_order.index(tup[0])) + + for name, field in items: + field.populate_obj(obj, name) + + +class Question(OrderedFieldForm): + + field_order = [ + 'answer', + 'clues', + 'week', + 'season', + 'imdb_url', + 'email_msg', + 'packshot', + ] + """A question form.""" def __init__(self, **kwargs): super(Question, self).__init__( **kwargs) @@ -218,7 +280,7 @@ def __init__(self, **kwargs): answer = FilmField('Film', [validators.Required()], id='film') clues = CluesFieldList(ClueFormField(ClueForm), min_entries=4) email_msg = fields.TextAreaField('Email Message') - packshot = ImageField('Image', 'questions') + packshot = PackshotImageField('Image', 'questions') imdb_url = fields.TextField('IMDB Link', default='http://www.imdb.com/title/XXX/') week = WeekField(choices=WeekField.week_choices()) @@ -230,17 +292,34 @@ class Registration(Form): username = fields.TextField('', [validate_username]) -class User(Form): +class User(OrderedFieldForm): + + field_order = [ + 'username', + 'email', + 'favourite_film', + 'pic', + ] + username = fields.TextField('', [validate_username]) email = fields.TextField(validators=[validators.Email()]) - pic = ImageField('pic', 'profiles') + pic = ProfileImageField('pic', 'profiles') favourite_film = FilmField() -class League(Form): +class League(OrderedFieldForm): + + field_order = [ + 'id', + 'name', + 'pic', + 'owner', + 'users', + ] + id = fields.HiddenField('') name = fields.TextField('', [validate_league_name]) - pic = ImageField('pic', 'leagues') + pic = LeagueImageField('pic', 'leagues') owner = CurrentUserField() users = LeagueUsersField() diff --git a/src/models.py b/src/models.py index 82a0c4f..5ab91de 100755 --- a/src/models.py +++ b/src/models.py @@ -247,7 +247,7 @@ def blob_from_url(url, username): logging.info('Saving file: %s' % file_name) gcs_file.write(result.content) gcs_file.close() - return blobstore.create_gs_key('/gs' + file_name) + return blobstore.BlobKey(blobstore.create_gs_key('/gs' + file_name)) @staticmethod def to_leaderboard_json(user): From dbdcdf9f807f0ab9f24edd4d7ed233c2af799574 Mon Sep 17 00:00:00 2001 From: Adam Mcgrath Date: Sun, 23 Aug 2015 14:46:36 +0100 Subject: [PATCH 07/10] Add some tests for adding a question --- src/tests/test_admin.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/src/tests/test_admin.py b/src/tests/test_admin.py index 521570f..0ecf9b4 100644 --- a/src/tests/test_admin.py +++ b/src/tests/test_admin.py @@ -12,10 +12,12 @@ import json import mock import unittest +import webtest import base import helpers import admin +import models class AdminTestCase(base.TestCase): @@ -88,5 +90,34 @@ def testPoseQuestionNoEmail(self): self.assertEqual(0, len(messages)) + def testAddQuestion(self): + helpers.user(email='foo@bar.com', is_trusted_tester=True).put() + os.environ['USER_IS_ADMIN'] = '1' + self.post('/admin/addquestion', params={ + 'answer': '/en/the_lost_boys', + 'season': '1', + 'week': '3', + 'clues-0-image': webtest.Upload('screenshot.jpg', 'screenshot contents'), + 'clues-1-text': 'clue 1', + 'clues-2-text': 'clue 2', + 'clues-3-text': 'clue 3', + 'email_msg': 'My email message', + 'imdb_url': 'http://www.imdb.com/title/foo/', + 'packshot': webtest.Upload('packshot.jpg', 'packshot contents'), + }, headers={'host': 'ffcapp.appspot.com'}) + question = models.Question.query().get() + season = models.Season.query().get() + clues = models.Clue.query().fetch(3) + + self.assertEqual('The Lost Boys', question.answer_title) + self.assertEqual(1, season.number) + self.assertEqual(3, question.week) + self.assertIn('/_ah/img/encoded_gs_file:', question.packshot_url()) + self.assertIn('/_ah/img/encoded_gs_file:', question.clue_image_url()) + self.assertEquals(3, len(clues)) + self.assertEquals('My email message', question.email_msg) + self.assertEquals('http://www.imdb.com/title/foo/', question.imdb_url) + + if __name__ == '__main__': unittest.main() \ No newline at end of file From 8e35981374f3be4b619e3f89db673f02a6882189 Mon Sep 17 00:00:00 2001 From: Adam Mcgrath Date: Sun, 23 Aug 2015 15:03:00 +0100 Subject: [PATCH 08/10] migrate leagues and users --- scripts/cloud_storage_migration.py | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/scripts/cloud_storage_migration.py b/scripts/cloud_storage_migration.py index ddad3dc..bd0db4f 100755 --- a/scripts/cloud_storage_migration.py +++ b/scripts/cloud_storage_migration.py @@ -83,18 +83,35 @@ def migrate_packshot(q): def migrate_questions(): - for q in models.Question.query().fetch(2): + for q in models.Question.query(): migrate_packshot(q) migrate_screenshot(q) +def migrate_users(): + for u in models.User.query(): + old_key = str(u.pic) + u.pic = save_image(u.pic, 'profiles', u.username_lower) + print ('profile: %s | old: %s | new: %s' % (u.username_lower, old_key, str(u.pic))) + u.put() + + +def migrate_leagues(): + for l in models.League.query(): + old_key = str(l.pic) + l.pic = save_image(l.pic, 'leagues', l.name_slug) + print ('league: %s | old: %s | new: %s' % (l.name_slug, old_key, str(l.pic))) + l.put() + + def main(): # Use 'localhost:8080' for dev server. remote_api_stub.ConfigureRemoteApi(APP_NAME, '/_ah/remote_api', auth_func, servername='ffcapp.appspot.com') - # migrate_questions() - # TODO leagues, users + migrate_questions() + migrate_users() + migrate_leagues() if __name__ == '__main__': From b84396b8e7c08b5b1df64c4a725b4f97d79a3582 Mon Sep 17 00:00:00 2001 From: Adam Mcgrath Date: Sun, 23 Aug 2015 15:16:55 +0100 Subject: [PATCH 09/10] fix form tests --- src/tests/test_forms.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/tests/test_forms.py b/src/tests/test_forms.py index 126d50b..6c602ad 100644 --- a/src/tests/test_forms.py +++ b/src/tests/test_forms.py @@ -136,12 +136,12 @@ def testFilmFieldPopulate(self): self.assertEqual(obj.foo_year, 'bar') self.assertEqual(obj.foo_title, 'baz') - # def testImageFieldPopulate(self): - # field = forms.ImageField().bind(Form(), 'a') - # self.req.GET['foo'] = 'bar' - # obj = mock.MagicMock() - # field.populate_obj(obj, 'baz') - # self.assertIsNotNone(obj.baz) + def testImageFieldPopulate(self): + field = forms.ImageField('foo', 'bar').bind(Form(), 'a') + self.req.GET['foo'] = 'bar' + obj = mock.MagicMock() + field.populate_obj(obj, 'baz') + self.assertIsNotNone(obj.baz) def testClueFieldListProcessNoClues(self): field_list = forms.CluesFieldList(FormField()).bind(Form(), 'a') From 5002750d17fc79384fe9b8174729a7891b90e5c3 Mon Sep 17 00:00:00 2001 From: Adam Mcgrath Date: Mon, 24 Aug 2015 07:43:37 +0100 Subject: [PATCH 10/10] Batch the script tasks [ci skip] --- scripts/cloud_storage_migration.py | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/scripts/cloud_storage_migration.py b/scripts/cloud_storage_migration.py index bd0db4f..ec2391e 100755 --- a/scripts/cloud_storage_migration.py +++ b/scripts/cloud_storage_migration.py @@ -45,6 +45,10 @@ def auth_func(): return (os.environ['USER_EMAIL'], getpass.getpass()) def save_image(blob_key, folder_name, file_name): + if not blob_key: + print 'ERROR: no blob %s/%s' % (folder_name, file_name) + return blob_key + img_file = blobstore.BlobInfo(blob_key) img_file_o = img_file.open() img_data = img_file_o.read() @@ -87,21 +91,33 @@ def migrate_questions(): migrate_packshot(q) migrate_screenshot(q) - -def migrate_users(): - for u in models.User.query(): +def migrate_users(curs=None): + users, next_curs, more = models.User.query().fetch_page(500, + start_cursor=curs) + for u in users: old_key = str(u.pic) u.pic = save_image(u.pic, 'profiles', u.username_lower) print ('profile: %s | old: %s | new: %s' % (u.username_lower, old_key, str(u.pic))) - u.put() + + ndb.put_multi(users) + + if more: + migrate_users(next_curs) -def migrate_leagues(): - for l in models.League.query(): +def migrate_leagues(curs=None): + leagues, next_curs, more = models.League.query().fetch_page(500, + start_cursor=curs) + + for l in leagues: old_key = str(l.pic) l.pic = save_image(l.pic, 'leagues', l.name_slug) print ('league: %s | old: %s | new: %s' % (l.name_slug, old_key, str(l.pic))) - l.put() + + ndb.put_multi(leagues) + + if more: + migrate_leagues(next_curs) def main():