Merge pull request #38 from WorldBank-Transport/feature/ec2_credentials

Update containers to play well with instance credentials

Author: olafveerman

README.md

@@ -17,7 +17,7 @@ docker build -t ram-analysis .
 ## Releasing a new version
 The process to release a new version:
 
-- still on `develop`, bump the version in `./ram-tools/package.json` and/or `./ram-analysis/package.json`
+- still on `develop`, bump the version in `./ram-vt/package.json` and/or `./ram-analysis/package.json`
 - set up PR, have somebody do a review and merge `develop` into `master`
 - CircleCI will add a new tag to git using the version in `package.json`  
 Because this repo holds two containers that are independently versioned, the git tags are prepended with the container name (eg. `ram-vt-v0.1.0`)

ram-analysis/.babelrc

@@ -0,0 +1,15 @@
+{
+ "presets": [
+    [
+      "@babel/preset-env",
+      {
+        "targets": {
+          "node": "current"
+        }
+      }
+    ]
+ ],
+ "plugins": [],
+  "sourceMaps": "inline",
+  "retainLines": true
+}

ram-analysis/.eslintrc

@@ -5,7 +5,7 @@
     "mocha": true
   },
   "parserOptions": {
-    "ecmaVersion": 6
+    "ecmaVersion": 2017
   },
   "rules": {
     "semi": [2, "always"],

ram-analysis/.nvmrc

@@ -0,0 +1 @@
+v6.15

ram-analysis/Dockerfile

@@ -4,7 +4,7 @@ COPY . /code
 
 WORKDIR /code
 
-RUN npm install
+RUN yarn
 
 RUN mkdir /conversion && cd /conversion && echo "disk=/var/tmp/stxxl,2500,memory" > .stxxl && ln -s /code/node_modules/osrm/profiles/lib/ .
 

ram-analysis/app/calculate-eta/README.md

@@ -1,6 +1,7 @@
 # Calculate ETA
 
-TODO: Add explanation about what the file does
+Calculates the time matrix between all origins and destinations in a given area.
+Executed as a forked node process that responds to messages.
 
 This file is not called directly because node does not support all the used es6 features.
 The file `calculateETA.js` in the parent directory contains the babel register and should be executed instead.

ram-analysis/app/calculateETA.js

@@ -1,7 +1,5 @@
 // only ES5 is allowed in this file
-require('babel-register')({
-  presets: [ 'es2015' ]
-});
+require('@babel/register')();
 
 // load the server
 require('./calculate-eta/');

ram-analysis/app/s3/index.js

@@ -1,7 +1,10 @@
 'use strict';
 import * as Minio from 'minio';
+import Http from 'http';
+import Https from 'https';
+
+import { getAWSInstanceCredentials } from '../utils/aws';
 
-var minioClient;
 const {
   STORAGE_HOST,
   STORAGE_PORT,
@@ -12,30 +15,64 @@ const {
   STORAGE_REGION
 } = process.env;
 
-switch (STORAGE_ENGINE) {
-  case 'minio':
-    minioClient = new Minio.Client({
-      endPoint: STORAGE_HOST,
-      port: parseInt(STORAGE_PORT),
-      secure: false,
-      accessKey: STORAGE_ACCESS_KEY,
-      secretKey: STORAGE_SECRET_KEY
-    });
-    break;
-  case 's3':
-    minioClient = new Minio.Client({
-      // Endpoint gets updated based on region.
-      endPoint: 's3.amazonaws.com',
-      region: STORAGE_REGION,
-      accessKey: STORAGE_ACCESS_KEY,
-      secretKey: STORAGE_SECRET_KEY
-    });
-    break;
-  default:
-    throw new Error('Invalid storage engine. Use s3 or minio');
-}
-
-export default minioClient;
-
 export const bucket = STORAGE_BUCKET;
 export const region = STORAGE_REGION;
+
+/**
+ * Initializes the minio s3 client depending on the engine and credentials
+ * source in use. Needs to be a promise because it may rely on asynchronously
+ * fetched credentials.
+ *
+ * @returns Minio Client
+ */
+export default async function S3 () {
+  let minioClient;
+  let agent;
+
+  switch (STORAGE_ENGINE) {
+    case 'minio':
+      minioClient = new Minio.Client({
+        endPoint: STORAGE_HOST,
+        port: parseInt(STORAGE_PORT),
+        secure: false,
+        accessKey: STORAGE_ACCESS_KEY,
+        secretKey: STORAGE_SECRET_KEY
+      });
+      agent = Http.globalAgent;
+      break;
+    case 's3':
+      let credentials;
+      if (!STORAGE_ACCESS_KEY && !STORAGE_SECRET_KEY) {
+        // If we're using a S3 storage engine but no accessKey and secretKey
+        // are set up, we assume that it is being run from a EC2 instance and
+        // will try to get the credentials through the url. We're not throwing
+        // any error if it fails because that is checked on startup.
+        // See app/index.js
+        const AWSInstanceCredentials = await getAWSInstanceCredentials();
+        credentials = {
+          accessKey: AWSInstanceCredentials.accessKey,
+          secretKey: AWSInstanceCredentials.secretKey,
+          sessionToken: AWSInstanceCredentials.sessionToken
+        };
+      } else {
+        credentials = {
+          accessKey: STORAGE_ACCESS_KEY,
+          secretKey: STORAGE_SECRET_KEY
+        };
+      }
+
+      minioClient = new Minio.Client({
+        endPoint: 's3.amazonaws.com',
+        ...credentials
+      });
+      agent = Https.globalAgent;
+      break;
+    default:
+      throw new Error('Invalid storage engine. Use s3 or minio');
+  }
+
+  // Temp fix for https://github.com/minio/minio-js/issues/641
+  minioClient.agent = agent;
+
+  return minioClient;
+}

ram-analysis/app/s3/utils.js

@@ -1,8 +1,9 @@
 'use strict';
-import s3, { bucket } from './';
+import S3, { bucket } from './';
 
 // Proxy of removeObject function, assuming the bucket.
-export function removeFile (file) {
+export async function removeFile (file) {
+  const s3 = await S3();
   return new Promise((resolve, reject) => {
     s3.removeObject(bucket, file, err => {
       if (err) {
@@ -14,7 +15,8 @@ export function removeFile (file) {
 }
 
 // Get file.
-export function getFile (file) {
+export async function getFile (file) {
+  const s3 = await S3();
   return new Promise((resolve, reject) => {
     s3.getObject(bucket, file, (err, dataStream) => {
       if (err) {
@@ -26,7 +28,8 @@ export function getFile (file) {
 }
 
 // Get file content.
-export function getFileContents (file) {
+export async function getFileContents (file) {
+  const s3 = await S3();
   return new Promise((resolve, reject) => {
     s3.getObject(bucket, file, (err, dataStream) => {
       if (err) return reject(err);
@@ -40,19 +43,14 @@ export function getFileContents (file) {
 }
 
 // Get file content in JSON.
-export function getJSONFileContents (file) {
-  return getFileContents(file)
-    .then(result => {
-      try {
-        return JSON.parse(result);
-      } catch (e) {
-        Promise.reject(e);
-      }
-    });
+export async function getJSONFileContents (file) {
+  const result = await getFileContents(file);
+  return JSON.parse(result);
 }
 
 // Get file and write to disk.
-export function writeFile (file, destination) {
+export async function writeFile (file, destination) {
+  const s3 = await S3();
   return new Promise((resolve, reject) => {
     s3.fGetObject(bucket, file, destination, err => {
       if (err) {
@@ -64,7 +62,8 @@ export function writeFile (file, destination) {
 }
 
 // Put file.
-export function putFile (file, data) {
+export async function putFile (file, data) {
+  const s3 = await S3();
   return new Promise((resolve, reject) => {
     s3.putObject(bucket, file, data, (err, etag) => {
       if (err) {

ram-analysis/app/utils/aws.js

@@ -0,0 +1,79 @@
+'use strict';
+import fetch from 'node-fetch';
+
+/**
+ * NOTE: This file is duplicated on some services. Be sure to update all of them
+ * - ram-analysis
+ * - ram-vt
+ * - ram-backend
+ */
+
+/**
+ * Cache for the credentials.
+ */
+let AWSInstanceCredentialsCache = {
+  accessKey: null,
+  secretKey: null,
+  sessionToken: null,
+  expireTime: null
+};
+
+/**
+ * Fetches the instance credentials for a given role name.
+ * The instance needs to belong to the given role.
+ *
+ * @param {string} roleName The role name to use when fetching the credentials
+ *
+ * @throws Error if any of the requests fail.
+ */
+export async function fetchAWSInstanceCredentials (roleName) {
+  // When inside a container in a ec2 instance (or when using fargate), the ecs
+  // client adds a varible with the credentials url. If is is available use that.
+  // Docs at: https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html
+  const relUrl = process.env.AWS_CONTAINER_CREDENTIALS_RELATIVE_URI;
+  let accessCredUrl = '';
+  if (relUrl) {
+    accessCredUrl = `http://169.254.170.2${relUrl}`;
+  } else {
+    // If we're inside an ec2 machine just use the regular url and fetch the
+    // role if it was not provided.
+    const awsIAMUrl = 'http://169.254.169.254/latest/meta-data/iam/security-credentials/';
+    if (!roleName) {
+      const roleRes = await fetch(awsIAMUrl, { timeout: 2000 });
+      if (roleRes.status >= 400) throw new Error('Unable to fetch role name');
+      roleName = await roleRes.text();
+    }
+    accessCredUrl = `${awsIAMUrl}${roleName}`;
+  }
+
+  const accessRes = await fetch(accessCredUrl, { timeout: 2000 });
+  if (accessRes.status >= 400) throw new Error('Unable to fetch access credentials');
+  const accessCredentials = await accessRes.json();
+
+  return {
+    accessKey: accessCredentials.AccessKeyId,
+    secretKey: accessCredentials.SecretAccessKey,
+    sessionToken: accessCredentials.Token,
+    // Set the expiration back 30min to give some margin.
+    expireTime: (new Date(accessCredentials.Expiration)).getTime() - 1800 * 1000
+  };
+}
+
+/**
+ * Gets the credentials from cache unless they are expired.
+ *
+ * @see fetchAWSInstanceCredentials()
+ *
+ * @param {string} roleName The role name to use when fetching the credentials.
+ * @param {bool} force Force fetching new credentials. Defaults to false.
+ */
+export async function getAWSInstanceCredentials (roleName, force = false) {
+  if (force) return fetchAWSInstanceCredentials(roleName);
+
+  if (Date.now() >= AWSInstanceCredentialsCache.expireTime) {
+    // Fetch new credentials.
+    AWSInstanceCredentialsCache = await fetchAWSInstanceCredentials(roleName);
+  }
+
+  return AWSInstanceCredentialsCache;
+}