feat(cli): skip existing devices

Author: coderbyheart

cli/commands/import-devices.ts

@@ -5,6 +5,7 @@ import { getAPISettings } from '@hello.nrfcloud.com/nrfcloud-api-helpers/setting
 import chalk from 'chalk'
 import { chunk } from 'lodash-es'
 import { table } from 'table'
+import { getByDeviceIds } from '../../devices/getByDeviceIds.js'
 import { compareLists } from '../../devices/import/compareLists.js'
 import { readDeviceCertificates } from '../../devices/import/readDeviceCertificates.js'
 import { readDevicesList } from '../../devices/import/readDevicesList.js'
@@ -79,20 +80,37 @@ export const importDevicesCommand = ({
 			process.exit(1)
 		}
 
+		const byIds = getByDeviceIds({ db, DevicesTableName: devicesTableName })
+		const existing = await byIds(
+			Array.from(devices.keys()).map((imei) => `oob-${imei}`),
+		)
+		const existingDevices = Object.keys(existing)
+
 		console.log(
 			table([
 				['Fingerprint', 'Device ID', 'Model', 'HW version'],
-				...Array.from(devices.entries()).map(
-					([imei, { fingerprint, hwVersion }]) => [
+				...Array.from(devices.entries())
+					.filter(([imei]) => !existingDevices.includes(`oob-${imei}`))
+					.map(([imei, { fingerprint, hwVersion }]) => [
 						chalk.green(fingerprint),
 						chalk.blue(imei),
 						chalk.blue(model),
 						chalk.blue(hwVersion),
-					],
-				),
+					]),
 			]),
 		)
 
+		if (existingDevices.length > 0) {
+			console.warn(
+				chalk.yellow(
+					`Skipping`,
+					existingDevices.length,
+					`devices which are already registered.`,
+				),
+			)
+			console.warn('')
+		}
+
 		if (dryRun === true) {
 			console.log(chalk.gray(`Dry run, not registering devices.`))
 			process.exit(2)
@@ -112,15 +130,17 @@ export const importDevicesCommand = ({
 		for (const page of chunk([...deviceCertificates.entries()], 1000)) {
 			// Bulk-ops API allows max 1,000 devices per request
 			const registration = await client.register(
-				Array.from(page).map(([imei, { certificate: certPem }]) => {
-					const deviceId = `oob-${imei}`
-					return {
-						deviceId,
-						subType: model.replace(/[^0-9a-z-]/gi, '-'),
-						tags: [model.replace(/[^0-9a-z-]/gi, ':')],
-						certPem,
-					}
-				}),
+				Array.from(page)
+					.filter(([imei]) => !existingDevices.includes(`oob-${imei}`))
+					.map(([imei, { certificate: certPem }]) => {
+						const deviceId = `oob-${imei}`
+						return {
+							deviceId,
+							subType: model.replace(/[^0-9a-z-]/gi, '-'),
+							tags: [model.replace(/[^0-9a-z-]/gi, ':')],
+							certPem,
+						}
+					}),
 			)
 
 			if ('error' in registration) {
@@ -140,7 +160,9 @@ export const importDevicesCommand = ({
 			devicesTableName,
 		})
 
-		for (const [imei, { fingerprint, hwVersion }] of devices.entries()) {
+		for (const [imei, { fingerprint, hwVersion }] of devices
+			.entries()
+			.filter(([imei]) => !existingDevices.includes(`oob-${imei}`))) {
 			const deviceId = `oob-${imei}`
 
 			const res = await r({

devices/getByDeviceIds.ts

@@ -0,0 +1,51 @@
+import {
+	type DynamoDBClient,
+	BatchGetItemCommand,
+} from '@aws-sdk/client-dynamodb'
+import { marshall } from '@aws-sdk/util-dynamodb'
+import { chunk } from 'lodash-es'
+import type { Device } from './device.js'
+import { toDevice } from './getDeviceById.js'
+
+export const getByDeviceIds =
+	({
+		db,
+		DevicesTableName,
+	}: {
+		db: DynamoDBClient
+		DevicesTableName: string
+	}) =>
+	async (
+		deviceIds: Array<string>,
+	): Promise<
+		Record<
+			string,
+			| {
+					device: Device
+			  }
+			| { error: Error }
+		>
+	> => {
+		const result: Record<string, { device: Device } | { error: Error }> = {}
+
+		for (const page of chunk(deviceIds, 100)) {
+			const res = await db.send(
+				new BatchGetItemCommand({
+					RequestItems: {
+						[DevicesTableName]: {
+							Keys: page.map((deviceId) => marshall({ deviceId })),
+						},
+					},
+				}),
+			)
+
+			if (res.Responses?.[DevicesTableName] === undefined) continue
+
+			for (const item of res.Responses[DevicesTableName]) {
+				const device = toDevice(item)
+				result[device.id] = { device }
+			}
+		}
+
+		return result
+	}

devices/getDeviceById.ts

@@ -30,24 +30,8 @@ export const getDeviceById =
 					error: new DeviceNotFoundError(deviceId),
 				}
 
-			const {
-				deviceId: id,
-				model,
-				account,
-				fingerprint,
-				hideDataBefore,
-			} = unmarshall(res.Item)
-			const device: Device = {
-				id,
-				fingerprint,
-				model,
-				account,
-			}
-			if (hideDataBefore !== undefined) {
-				device.hideDataBefore = new Date(hideDataBefore)
-			}
 			return {
-				device,
+				device: toDevice(res.Item),
 			}
 		} catch (error) {
 			return {
@@ -56,6 +40,26 @@ export const getDeviceById =
 		}
 	}
 
+export const toDevice = (item: Record<string, any>): Device => {
+	const {
+		deviceId: id,
+		model,
+		account,
+		fingerprint,
+		hideDataBefore,
+	} = unmarshall(item)
+	const device: Device = {
+		id,
+		fingerprint,
+		model,
+		account,
+	}
+	if (hideDataBefore !== undefined) {
+		device.hideDataBefore = new Date(hideDataBefore)
+	}
+	return device
+}
+
 export class DeviceNotFoundError extends Error {
 	public readonly id: string
 	constructor(id: string) {