Check database is ready before running tests

In CircleCI, the tests are running before MySQL is ready to take
connections, leading to many test failures. Wait for the database to be
ready before running tests.

bin/test_env.sh

@@ -80,7 +80,22 @@ docker run \
        --entrypoint="" \
        "${TESTIMAGE}" chown -R "${ICHNAEA_UID}:${ICHNAEA_GID}" /app
 
-# Run cmd in that environment and then remove the container
+# Check that database server is ready for tests
+docker run \
+    --rm \
+    --user "${ICHNAEA_UID}" \
+    --volumes-from ichnaea-repo \
+    --workdir /app \
+    --network ichnaea_default \
+    --link ichnaea_db_1 \
+    --link ichnaea_redis_1 \
+    --env-file ./docker/config/local_dev.env \
+    --tty \
+    --interactive \
+    --entrypoint= \
+    "${TESTIMAGE}" /app/docker/run_check_db.sh
+
+# Run tests in that environment and then remove the container
 echo "Running tests..."
 docker run \
     --rm \

docker/mysql_5_7/Dockerfile

@@ -2,4 +2,5 @@ FROM mysql:5.7.27
 ENV MYSQL_ROOT_PASSWORD=location
 ENV MYSQL_USER=location
 ENV MYSQL_PASSWORD=location
+ENV MYSQL_DATABASE=location
 COPY my.cnf /etc/mysql/conf.d/my.cnf

docker/run_check_db.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+set -e
+
+# Check that the database is running
+MAX_ATTEMPTS=6
+ATTEMPT=1
+until python ichnaea/scripts/db.py check
+do
+    if ((ATTEMPT==MAX_ATTEMPTS))
+    then
+        echo "Database connection failed."
+        exit 1
+    else
+        echo "Database is not yet ready. Trying again in 5 seconds."
+        sleep 5
+        ATTEMPT=$((ATTEMPT+1))
+    fi
+done

ichnaea/db.py

@@ -275,6 +275,15 @@ def drop_db(uri=None):
         conn.execute(f"DROP DATABASE IF EXISTS {db_to_drop}")
 
 
+def database_is_ready():
+    """Raises OperationalError if database fails."""
+    uri = get_sqlalchemy_url()
+    db = create_engine(uri)
+    db.connect()
+    db.execute("SELECT 1;")
+    return True
+
+
 def retry_on_mysql_lock_fail(metric=None, metric_tags=None):
     """Function decorator to backoff and retry on MySQL lock failures.
 

ichnaea/scripts/db.py

@@ -8,9 +8,9 @@ Use this only in a local dev environment.
 import argparse
 import sys
 
-from sqlalchemy.exc import ProgrammingError
+from sqlalchemy.exc import OperationalError, ProgrammingError
 
-from ichnaea.db import drop_db, create_db, get_sqlalchemy_url
+from ichnaea.db import create_db, database_is_ready, drop_db, get_sqlalchemy_url
 
 
 def create_database():
@@ -33,12 +33,25 @@ def drop_database():
     print("Done.")
 
 
+def check_database():
+    """Check if the database is ready for connections."""
+    try:
+        database_is_ready()
+    except OperationalError as e:
+        print(f"Database is not ready: {e}")
+        return 1
+    else:
+        print("Database is ready.")
+        return 0
+
+
 def main(argv):
-    parser = argparse.ArgumentParser(description="Create and delete databases.")
+    parser = argparse.ArgumentParser(description="Work with databases.")
     subparsers = parser.add_subparsers(dest="cmd")
     subparsers.required = True
     subparsers.add_parser("drop", help="drop existing database")
     subparsers.add_parser("create", help="create database")
+    subparsers.add_parser("check", help="check if database is accepting connections")
 
     args = parser.parse_args(argv[1:])
 
@@ -46,6 +59,8 @@ def main(argv):
         return drop_database()
     elif args.cmd == "create":
         return create_database()
+    elif args.cmd == "check":
+        return check_database()
 
 
 if __name__ == "__main__":