Merge pull request #39 from WorldBank-Transport/develop

Release new versions

Author: olafveerman

.circleci/config.yml

@@ -95,6 +95,10 @@ jobs:
             docker tag ${DOCKER_IMAGE} ${DOCKER_ORG}/${DOCKER_IMAGE}:${VERSION}
             docker push ${DOCKER_ORG}/${DOCKER_IMAGE}:${VERSION}
 
+            echo Pushing image to Docker Hub with latest tag
+            docker tag ${DOCKER_IMAGE} ${DOCKER_ORG}/${DOCKER_IMAGE}:latest
+            docker push ${DOCKER_ORG}/${DOCKER_IMAGE}:latest
+
   deploy-analysis-prod:
     docker: 
       - image: circleci/node:6
@@ -152,6 +156,10 @@ jobs:
             docker tag ${DOCKER_IMAGE} ${DOCKER_ORG}/${DOCKER_IMAGE}:${VERSION}
             docker push ${DOCKER_ORG}/${DOCKER_IMAGE}:${VERSION}
 
+            echo Pushing image to Docker Hub with latest tag
+            docker tag ${DOCKER_IMAGE} ${DOCKER_ORG}/${DOCKER_IMAGE}:latest
+            docker push ${DOCKER_ORG}/${DOCKER_IMAGE}:latest
+
 workflows:
   version: 2
   build-deploy:

README.md

@@ -17,7 +17,7 @@ docker build -t ram-analysis .
 ## Releasing a new version
 The process to release a new version:
 
-- still on `develop`, bump the version in `./ram-tools/package.json` and/or `./ram-analysis/package.json`
+- still on `develop`, bump the version in `./ram-vt/package.json` and/or `./ram-analysis/package.json`
 - set up PR, have somebody do a review and merge `develop` into `master`
 - CircleCI will add a new tag to git using the version in `package.json`  
 Because this repo holds two containers that are independently versioned, the git tags are prepended with the container name (eg. `ram-vt-v0.1.0`)

ram-analysis/.babelrc

@@ -0,0 +1,15 @@
+{
+ "presets": [
+    [
+      "@babel/preset-env",
+      {
+        "targets": {
+          "node": "current"
+        }
+      }
+    ]
+ ],
+ "plugins": [],
+  "sourceMaps": "inline",
+  "retainLines": true
+}

ram-analysis/.eslintrc

@@ -5,7 +5,7 @@
     "mocha": true
   },
   "parserOptions": {
-    "ecmaVersion": 6
+    "ecmaVersion": 2017
   },
   "rules": {
     "semi": [2, "always"],

ram-analysis/.nvmrc

@@ -0,0 +1 @@
+v6.15

ram-analysis/Dockerfile

@@ -4,7 +4,7 @@ COPY . /code
 
 WORKDIR /code
 
-RUN npm install
+RUN yarn
 
 RUN mkdir /conversion && cd /conversion && echo "disk=/var/tmp/stxxl,2500,memory" > .stxxl && ln -s /code/node_modules/osrm/profiles/lib/ .
 

ram-analysis/app/calculate-eta/README.md

@@ -1,6 +1,7 @@
 # Calculate ETA
 
-TODO: Add explanation about what the file does
+Calculates the time matrix between all origins and destinations in a given area.
+Executed as a forked node process that responds to messages.
 
 This file is not called directly because node does not support all the used es6 features.
 The file `calculateETA.js` in the parent directory contains the babel register and should be executed instead.

ram-analysis/app/calculateETA.js

@@ -1,7 +1,5 @@
 // only ES5 is allowed in this file
-require('babel-register')({
-  presets: [ 'es2015' ]
-});
+require('@babel/register')();
 
 // load the server
 require('./calculate-eta/');

ram-analysis/app/s3/index.js

@@ -1,7 +1,10 @@
 'use strict';
 import * as Minio from 'minio';
+import Http from 'http';
+import Https from 'https';
+
+import { getAWSInstanceCredentials } from '../utils/aws';
 
-var minioClient;
 const {
   STORAGE_HOST,
   STORAGE_PORT,
@@ -12,30 +15,64 @@ const {
   STORAGE_REGION
 } = process.env;
 
-switch (STORAGE_ENGINE) {
-  case 'minio':
-    minioClient = new Minio.Client({
-      endPoint: STORAGE_HOST,
-      port: parseInt(STORAGE_PORT),
-      secure: false,
-      accessKey: STORAGE_ACCESS_KEY,
-      secretKey: STORAGE_SECRET_KEY
-    });
-    break;
-  case 's3':
-    minioClient = new Minio.Client({
-      // Endpoint gets updated based on region.
-      endPoint: 's3.amazonaws.com',
-      region: STORAGE_REGION,
-      accessKey: STORAGE_ACCESS_KEY,
-      secretKey: STORAGE_SECRET_KEY
-    });
-    break;
-  default:
-    throw new Error('Invalid storage engine. Use s3 or minio');
-}
-
-export default minioClient;
-
 export const bucket = STORAGE_BUCKET;
 export const region = STORAGE_REGION;
+
+/**
+ * Initializes the minio s3 client depending on the engine and credentials
+ * source in use. Needs to be a promise because it may rely on asynchronously
+ * fetched credentials.
+ *
+ * @returns Minio Client
+ */
+export default async function S3 () {
+  let minioClient;
+  let agent;
+
+  switch (STORAGE_ENGINE) {
+    case 'minio':
+      minioClient = new Minio.Client({
+        endPoint: STORAGE_HOST,
+        port: parseInt(STORAGE_PORT),
+        secure: false,
+        accessKey: STORAGE_ACCESS_KEY,
+        secretKey: STORAGE_SECRET_KEY
+      });
+      agent = Http.globalAgent;
+      break;
+    case 's3':
+      let credentials;
+      if (!STORAGE_ACCESS_KEY && !STORAGE_SECRET_KEY) {
+        // If we're using a S3 storage engine but no accessKey and secretKey
+        // are set up, we assume that it is being run from a EC2 instance and
+        // will try to get the credentials through the url. We're not throwing
+        // any error if it fails because that is checked on startup.
+        // See app/index.js
+        const AWSInstanceCredentials = await getAWSInstanceCredentials();
+        credentials = {
+          accessKey: AWSInstanceCredentials.accessKey,
+          secretKey: AWSInstanceCredentials.secretKey,
+          sessionToken: AWSInstanceCredentials.sessionToken
+        };
+      } else {
+        credentials = {
+          accessKey: STORAGE_ACCESS_KEY,
+          secretKey: STORAGE_SECRET_KEY
+        };
+      }
+
+      minioClient = new Minio.Client({
+        endPoint: 's3.amazonaws.com',
+        ...credentials
+      });
+      agent = Https.globalAgent;
+      break;
+    default:
+      throw new Error('Invalid storage engine. Use s3 or minio');
+  }
+
+  // Temp fix for https://github.com/minio/minio-js/issues/641
+  minioClient.agent = agent;
+
+  return minioClient;
+}

ram-analysis/app/s3/utils.js

@@ -1,8 +1,9 @@
 'use strict';
-import s3, { bucket } from './';
+import S3, { bucket } from './';
 
 // Proxy of removeObject function, assuming the bucket.
-export function removeFile (file) {
+export async function removeFile (file) {
+  const s3 = await S3();
   return new Promise((resolve, reject) => {
     s3.removeObject(bucket, file, err => {
       if (err) {
@@ -14,7 +15,8 @@ export function removeFile (file) {
 }
 
 // Get file.
-export function getFile (file) {
+export async function getFile (file) {
+  const s3 = await S3();
   return new Promise((resolve, reject) => {
     s3.getObject(bucket, file, (err, dataStream) => {
       if (err) {
@@ -26,7 +28,8 @@ export function getFile (file) {
 }
 
 // Get file content.
-export function getFileContents (file) {
+export async function getFileContents (file) {
+  const s3 = await S3();
   return new Promise((resolve, reject) => {
     s3.getObject(bucket, file, (err, dataStream) => {
       if (err) return reject(err);
@@ -40,19 +43,14 @@ export function getFileContents (file) {
 }
 
 // Get file content in JSON.
-export function getJSONFileContents (file) {
-  return getFileContents(file)
-    .then(result => {
-      try {
-        return JSON.parse(result);
-      } catch (e) {
-        Promise.reject(e);
-      }
-    });
+export async function getJSONFileContents (file) {
+  const result = await getFileContents(file);
+  return JSON.parse(result);
 }
 
 // Get file and write to disk.
-export function writeFile (file, destination) {
+export async function writeFile (file, destination) {
+  const s3 = await S3();
   return new Promise((resolve, reject) => {
     s3.fGetObject(bucket, file, destination, err => {
       if (err) {
@@ -64,7 +62,8 @@ export function writeFile (file, destination) {
 }
 
 // Put file.
-export function putFile (file, data) {
+export async function putFile (file, data) {
+  const s3 = await S3();
   return new Promise((resolve, reject) => {
     s3.putObject(bucket, file, data, (err, etag) => {
       if (err) {