airgradient

.env

@@ -0,0 +1,5 @@
+LCS_API=https://api.openaq.org
+STACK=my-dev-stack
+SECRET_STACK=my-prod-stack
+BUCKET=openaq-fetches
+VERBOSE=1

fetcher/providers/airgradient.js

@@ -0,0 +1,138 @@
+const Providers = require('../lib/providers');
+const { Sensor, SensorNode, SensorSystem } = require('../lib/station');
+const { Measures, FixedMeasure } = require('../lib/measure');
+const { Measurand } = require('../lib/measurand');
+const { fetchSecret, request } = require('../lib/utils');
+const dayjs = require('dayjs');
+const utc = require('dayjs/plugin/utc');
+const { find } = require('geo-tz');
+
+
+dayjs.extend(utc);
+
+const lookup = {
+    // input_param: [measurand_parameter, measurand_unit]
+    'pm02': ['pm25', 'µg/m³'],
+    'atmp': ['temperature', 'c']
+};
+
+
+async function getDevices(source, token) {
+    const url = new URL('/public/api/v1/world/locations/measures/current', source.meta.url);
+    url.searchParams.append('token', token);
+    const { body } = await request({
+        json: true,
+        method: 'GET',
+        url: url
+    });
+    return body;
+}
+
+
+async function getAllSensors(source, devices, token) {
+    return Promise.all(devices.map(async (device) => {
+        const { locationId } = device;
+        const sensorData = await fetchSensorData(source, locationId, token);
+        device.measurements = getLatestReading(sensorData);
+        return device;
+    }));
+}
+
+function getLatestReading(sensorData) {
+    const d = new Date();
+    d.setHours(d.getHours() - 1);
+    d.setMinutes(0);
+    d.setSeconds(0);
+    d.setMilliseconds(0);
+    const currentMeasurement = sensorData.filter((o) => new Date(o.date).getTime() === d.getTime());
+    const timestamp = new Date(currentMeasurement[0].date);
+    timestamp.setHours(timestamp.getHours() + 1); // convert to hour ending
+    const params = ['date'];
+    const measurements = Object.entries(currentMeasurement[0]).map(([k,v]) => {
+        if (!params.includes(k)) {
+            return { 'parameter': k, 'value': v, 'timestamp': timestamp.toISOString() };
+        }
+    }).filter((o) => o !== undefined);
+    return measurements;
+}
+
+async function processor(source_name, source) {
+    const token = await fetchSecret('airgradient');
+    let devices  = await getDevices(source, token);
+    devices = devices.filter((o) => !o.offline);
+    devices = devices.filter((o) => o.latitude != null && o.longitude != null );
+    const measurands = await Measurand.getIndexedSupportedMeasurands(lookup);
+    const readings = await getAllSensors(source, devices, token);
+    const stations = readings.map((reading) => {
+        const sensorNode = new SensorNode({
+            sensor_node_id: `airgradient-${reading.locationId}`,
+            sensor_node_source_name: `${source_name}`,
+            sensor_node_site_name: reading.name,
+            sensor_node_geometry: [reading.longitude, reading.latitude],
+            sensor_node_city: reading.city,
+            sensor_node_country: reading.country,
+            sensor_node_timezone: find(reading.latitude, reading.longitude)[0],
+            sensor_node_ismobile: false,
+            sensor_system: new SensorSystem({
+                sensor_system_manufacturer_name: 'AirGradient',
+                sensors: reading.measurements
+                    .map((o) => { return measurands[o.parameter];})
+                    .filter(Boolean)
+                    .map((measurand) =>
+                        new Sensor({
+                            sensor_id: `airgradient-${reading.locationId}-${measurand.parameter}`,
+                            measurand_parameter: measurand.parameter,
+                            measurand_unit: measurand.normalized_unit
+                        })
+                    )
+            })
+        });
+        return Providers.put_station(source_name, sensorNode);
+    });
+
+    const measures = new Measures(FixedMeasure);
+
+    readings.map((reading) => {
+        if (reading.measurements) {
+            reading.measurements.map((o) => {
+                const measurand = measurands[o.parameter];
+                (!measurand) ? [] :
+                    measures.push({
+                        sensor_id: `airgradient-${reading.locationId}-${measurand.parameter}`,
+                        measure: measurand.normalize_value(o.value),
+                        timestamp: o.timestamp
+                    });
+            });
+        }
+
+    });
+    console.log(`ok - all ${stations.length} stations pushed`);
+    Providers.put_measures(source_name, measures, `airgradient-${Math.floor(Date.now() / 1000)}-${Math.random().toString(36).substring(8)}`);
+    console.log(`ok - all ${measures.length} measurements pushed`);
+}
+
+
+async function fetchSensorData(source, locationId, token) {
+    // /world/locations/{locationId}/measures/last/buckets/{period}
+    const url = new URL(`/public/api/v1/world/locations/${locationId}/measures/last/buckets/60`, source.meta.url);
+    url.searchParams.append('token', token);
+    let response = {};
+    const { body, statusCode } = await request({
+        json: true,
+        method: 'GET',
+        url: url
+    });
+    if (statusCode === 200) {
+        response = body;
+        body.locationId = locationId;
+        response.statusCode = 200;
+    } else {
+        response.statusCode = statusCode;
+    }
+    return response;
+}
+
+
+module.exports = {
+    processor
+};

fetcher/providers/clarity.js

@@ -62,13 +62,13 @@ class ClarityApi {
             let ds = response.body;
 
             if (process.env.SOURCEID) {
-              ds = ds.filter((d) => d.deviceCode === process.env.SOURCEID);
+                ds = ds.filter((d) => d.deviceCode === process.env.SOURCEID);
             } else {
-              ds = ds.filter(d=>d.sourceType == 'CLARITY_NODE');
+                ds = ds.filter((d)=>d.sourceType === 'CLARITY_NODE');
             }
 
             if (VERBOSE) {
-              console.debug(`-------------------\nListing ${ds.length} sources for ${this.org.organizationName}`);
+                console.debug(`-------------------\nListing ${ds.length} sources for ${this.org.organizationName}`);
                 ds.map((d) => console.log(`${d.sourceType}: (${d.subscriptionStatus}) ${d.deviceCode} - ${d.name}`));
             }
             return ds;
@@ -79,12 +79,12 @@ class ClarityApi {
      *
      * @returns {Promise<Device[]>}
      */
-  listDevices() {
-    return request({
-      json: true,
-      method: 'GET',
-      headers: { 'X-API-Key': this.apiKey },
-      url: new URL('v2/devices/nodes', this.baseUrl),
+    listDevices() {
+        return request({
+            json: true,
+            method: 'GET',
+            headers: { 'X-API-Key': this.apiKey },
+            url: new URL('v2/devices/nodes', this.baseUrl),
             qs: { 'org': this.orgId }
         }).then((response) => response.body).then((response) => {
             if (process.env.SOURCEID) {
@@ -95,7 +95,7 @@ class ClarityApi {
             }
             const working = response.filter((o) => o.lifeStage.stage === 'working');
             if (VERBOSE) {
-              console.debug(`-----------------\nListing devices for ${this.org.organizationName}\nFound ${response.length} total devices, ${working.length} working`);
+                console.debug(`-----------------\nListing devices for ${this.org.organizationName}\nFound ${response.length} total devices, ${working.length} working`);
                 response
                     .filter((d) => d.lifeStage.stage !== 'working')
                     .map((d) => console.log(`DEVICE: ${d.nodeId} - ${d.model} - ${d.lifeStage.stage}`));
@@ -152,11 +152,11 @@ class ClarityApi {
         while (true) {
             url.searchParams.set('skip', offset);
             if (VERBOSE) console.log(`Fetching ${url}&key=${this.apiKey}`);
-          const response = await request({
-            url,
+            const response = await request({
+                url,
                 json: true,
                 method: 'GET',
-              headers: { 'X-API-Key': this.apiKey, 'Accept-Encoding': 'gzip' },
+                headers: { 'X-API-Key': this.apiKey, 'Accept-Encoding': 'gzip' },
                 gzip: true
             });
 
@@ -196,42 +196,42 @@ class ClarityApi {
      */
     async sync(supportedMeasurands, since) {
         // get all the devices, even if expired
-      let devices = await this.listAugmentedDevices();
-
-      if (VERBOSE) {
-        console.debug(`-----------------------\n Syncing ${this.source.provider}/${this.org.organizationName}`, devices.length);
-        devices.map( d => {
-          if(d.pairedAccessoryModules.length>0) {
-            console.debug(`--------------------\n Found device with ${d.length} modules`);
-          }
-        });
-      }
+        let devices = await this.listAugmentedDevices();
+
+        if (VERBOSE) {
+            console.debug(`-----------------------\n Syncing ${this.source.provider}/${this.org.organizationName}`, devices.length);
+            devices.map( (d) => {
+                if (d.pairedAccessoryModules.length > 0) {
+                    console.debug(`--------------------\n Found device with ${d.length} modules`);
+                }
+            });
+        }
         // Create one station per device
         const stations = devices.map((device) =>
             Providers.put_station(
                 this.source.provider,
-              new SensorNode({
-                sensor_node_id: `${this.org.organizationName}-${device.nodeId}`,
-                sensor_node_site_name: device.name || device.nodeId, // fall back to code when missing name
-                sensor_node_geometry: device.location.coordinates,
-                sensor_node_status: device.subscriptionStatus,
-                sensor_node_source_name: this.source.provider, //
-                sensor_node_site_description: this.org.organizationName,
-                  sensor_node_ismobile: false, // should remove this and just use instrument
-                  //sensor_node_instrument: device.model,
+                new SensorNode({
+                    sensor_node_id: `${this.org.organizationName}-${device.nodeId}`,
+                    sensor_node_site_name: device.name || device.nodeId, // fall back to code when missing name
+                    sensor_node_geometry: device.location.coordinates,
+                    sensor_node_status: device.subscriptionStatus,
+                    sensor_node_source_name: this.source.provider, //
+                    sensor_node_site_description: this.org.organizationName,
+                    sensor_node_ismobile: false, // should remove this and just use instrument
+                    // sensor_node_instrument: device.model,
                     sensor_node_deployed_date: device.lifeStage.when,
                     sensor_system: new SensorSystem({
                         sensor_system_manufacturer_name: this.source.provider,
                         //   Create one sensor per characteristic
-                      sensors: [] //Object.values(supportedMeasurands)
-                        .map(
-                          (measurand) =>
-                          new Sensor({
-                            sensor_id: getSensorId(device, measurand),
-                            measurand_parameter: measurand.parameter,
-                            measurand_unit: measurand.normalized_unit
-                          })
-                        )
+                        sensors: [] // Object.values(supportedMeasurands)
+                            .map(
+                                (measurand) =>
+                                    new Sensor({
+                                        sensor_id: getSensorId(device, measurand),
+                                        measurand_parameter: measurand.parameter,
+                                        measurand_unit: measurand.normalized_unit
+                                    })
+                            )
                     })
                 })
             )

fetcher/providers/habitatmap.js

@@ -127,7 +127,7 @@ async function process_mobile_locations(source_name, source, measurands) {
 
 async function fixed_locations(source) {
     const params = {
-      time_from: String(Math.round(Date.now() / 1000) - 60 * 2), // 60s * 2min
+        time_from: String(Math.round(Date.now() / 1000) - 60 * 2), // 60s * 2min
         time_to: String(Math.round(Date.now() / 1000)),
         tags: '',
         usernames: '',

fetcher/sources/airgradient.json

@@ -0,0 +1,9 @@
+{
+    "schema": "v1",
+    "provider": "airgradient",
+    "frequency": "hour",
+    "meta": {
+      "url": "https://api.airgradient.com"
+    }
+  }
+