-
Notifications
You must be signed in to change notification settings - Fork 2
/
draft-irtf-t2trg-iot-edge.xml
2692 lines (2677 loc) · 177 KB
/
draft-irtf-t2trg-iot-edge.xml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
<?xml version='1.0' encoding='utf-8'?>
<!DOCTYPE rfc [
<!ENTITY nbsp " ">
<!ENTITY zwsp "​">
<!ENTITY nbhy "‑">
<!ENTITY wj "⁠">
]>
<?xml-stylesheet type="text/xsl" href="rfc2629.xslt" ?>
<!-- generated by https://github.com/cabo/kramdown-rfc version 1.6.39 (Ruby 3.0.2) -->
<?rfc iprnotified="yes"?>
<?rfc strict="yes"?>
<?rfc compact="yes"?>
<?rfc colonspace="yes"?>
<?rfc rfcedstyle="no"?>
<rfc xmlns:xi="http://www.w3.org/2001/XInclude" ipr="trust200902" docName="draft-irtf-t2trg-iot-edge-10" category="info" submissionType="IRTF" tocDepth="4" tocInclude="true" sortRefs="true" symRefs="true" version="3">
<!-- xml2rfc v2v3 conversion 3.18.0 -->
<front>
<title abbrev="IoT Edge Computing">IoT Edge Challenges and Functions</title>
<seriesInfo name="Internet-Draft" value="draft-irtf-t2trg-iot-edge-10"/>
<author initials="J." surname="Hong" fullname="Jungha Hong">
<organization>ETRI</organization>
<address>
<postal>
<street>218 Gajeong-ro, Yuseung-Gu</street>
<city>Daejeon</city>
<code>34129</code>
<country>Republic of Korea</country>
</postal>
<email>[email protected]</email>
</address>
</author>
<author initials="Y.-G." surname="Hong" fullname="Yong-Geun Hong">
<organization>Daejeon University</organization>
<address>
<postal>
<street>62 Daehak-ro, Dong-gu</street>
<city>Daejeon</city>
<code>300716</code>
<country>Republic of Korea</country>
</postal>
<email>[email protected]</email>
</address>
</author>
<author initials="X." surname="de Foy" fullname="Xavier de Foy">
<organization>InterDigital Communications, LLC</organization>
<address>
<postal>
<street>1000 Sherbrooke West</street>
<city>Montreal</city>
<code>H3A 3G4</code>
<country>Canada</country>
</postal>
<email>[email protected]</email>
</address>
</author>
<author initials="M." surname="Kovatsch" fullname="Matthias Kovatsch">
<organization>Huawei Technologies Duesseldorf GmbH</organization>
<address>
<postal>
<street>Riesstr. 25 C // 3.OG</street>
<city>Munich</city>
<code>80992</code>
<country>Germany</country>
</postal>
<email>[email protected]</email>
</address>
</author>
<author initials="E." surname="Schooler" fullname="Eve Schooler">
<organization>Intel</organization>
<address>
<postal>
<street>2200 Mission College Blvd.</street>
<city>Santa Clara, CA</city>
<code>95054-1537</code>
<country>USA</country>
</postal>
<email>[email protected]</email>
</address>
</author>
<author initials="D." surname="Kutscher" fullname="Dirk Kutscher">
<organization>Hong Kong University of Science and Technology (Guangzhou)</organization>
<address>
<postal>
<street>No.1 Du Xue Rd</street>
<city>Guangzhou</city>
<country>China</country>
</postal>
<email>[email protected]</email>
</address>
</author>
<date year="2023" month="September" day="15"/>
<area>T2TRG</area>
<abstract>
<?line 879?>
<t>Many Internet of Things (IoT) applications have requirements that cannot be satisfied by traditional cloud-based systems (i.e., cloud computing). These include time sensitivity, data volume, connectivity cost, operation in the face of intermittent services, privacy, and security. As a result, IoT is driving the Internet toward edge computing. This document outlines the requirements of the emerging IoT Edge and its challenges. It presents a general model and major components of the IoT Edge to provide a common basis for future discussions in the T2TRG and other IRTF and IETF groups. This document is a product of the IRTF Thing-to-Thing Research Group (T2TRG).</t>
</abstract>
</front>
<middle>
<?line 883?>
<section anchor="introduction">
<name>Introduction</name>
<t>Currently, many IoT services leverage cloud computing platforms, because they provide virtually unlimited storage and processing power. The reliance of IoT on back-end cloud computing provides additional advantages such as scalability and efficiency. Today's IoT systems are fairly static with respect to integrating and supporting computation. It is not that there is no computation, but that systems are often limited to static configurations (edge gateways and cloud services).</t>
<t>However, IoT devices generate large amounts of data at the edges of the network. To meet IoT use case requirements, data is increasingly being stored, processed, analyzed, and acted upon close to the data sources. These requirements include time sensitivity, data volume, connectivity cost, and resiliency in the presence of intermittent connectivity, privacy, and security, which cannot be addressed by centralized cloud computing. A more flexible approach is necessary to address these needs effectively. This involves distributing computing (and storage) and seamlessly integrating it into the edge-cloud continuum. We refer to this integration of edge computing and IoT as "IoT edge computing". This draft describes the related background, use cases, challenges, system models, and functional components.</t>
<t>Owing to the dynamic nature of the IoT edge computing landscape, this document does not list existing projects in this field. <xref target="sec-overview"/> presents a high-level overview of the field, based on a limited review of standards, research, open-source and proprietary products in <xref target="I-D.defoy-t2trg-iot-edge-computing-background"/>.</t>
<t>This document represents the consensus of the Thing-to-Thing Research Group (T2TRG). It has been reviewed extensively by the Research Group (RG) members who are actively involved in the research and development of the technology covered by this document. It is not an IETF product and is not a standard.</t>
</section>
<section anchor="background">
<name>Background</name>
<section anchor="internet-of-things-iot">
<name>Internet of Things (IoT)</name>
<t>Since the term "Internet of Things" (IoT) was coined by Kevin Ashton in 1999 working on Radio-Frequency Identification (RFID) technology <xref target="Ashton"/>, the concept of IoT has evolved. It now reflects a vision of connecting the physical world to the virtual world of computers using (often wireless) networks over which things can send and receive information without human intervention. Recently, the term has become more literal by connecting things to the Internet and converging on Internet and Web technologies.</t>
<t>A Thing is a physical item made available in the IoT, thereby enabling digital interaction with the physical world for humans, services, and/or other Things (<xref target="I-D.irtf-t2trg-rest-iot"/>). In this document we will use the term "IoT device" to designate the embedded system attached to the Thing.</t>
<t>Resource-constrained Things such as sensors, home appliances and wearable devices often have limited storage and processing power, which can provide challenges with respect to reliability, performance, energy consumption, security, and privacy <xref target="Lin"/>. Some, less resource-constrained Things, can generate a voluminous amount of data. This range of factors led IoT designs that integrate Things into larger distributed systems, for example edge or cloud computing systems.</t>
</section>
<section anchor="cloud-computing">
<name>Cloud Computing</name>
<t>Cloud computing has been defined in <xref target="NIST"/>: "cloud computing is a model for enabling ubiquitous, convenient, on-demand network access to a shared pool of configurable computing resources (e.g., networks, servers, storage, applications, and services) that can be rapidly provisioned and released with minimal management effort or service provider interaction". The low cost and massive availability of storage and processing power enabled the realization of another computing model, in which virtualized resources can be leased in an on-demand fashion and be provided as general utilities. Platform-as-a-Service and cloud computing platforms widely adopted this paradigm for delivering services over the Internet, gaining both economical and technical benefits <xref target="Botta"/>.</t>
<t>Today, an unprecedented volume and variety of data is generated by Things, and applications deployed at the network edge consume this data. In this context, cloud-based service models are not suitable for some classes of applications which require very short response times, access to local personal data, or generate vast amounts of data. These applications may instead leverage edge computing.</t>
</section>
<section anchor="edge-computing">
<name>Edge Computing</name>
<t>Edge computing, also referred to as fog computing in some settings, is a new paradigm in which substantial computing and storage resources are placed at the edge of the Internet, close to mobile devices, sensors, actuators, or machines. Edge computing happens near data sources <xref target="Mahadev"/>, as well as close to where decisions are made or where interactions with the physical world take place ("close" here can refer to a distance which is topological, physical, latency-based, etc.). It processes both downstream data (originating from cloud services) and upstream data (originating from end devices or network elements). The term "fog computing" usually represents the notion of multi-tiered edge computing, that is, several layers of compute infrastructure between end devices and cloud services.</t>
<t>An edge device is any computing or networking resource residing between end-device data sources and cloud-based data centers. In edge computing, end devices consume and produce data. At the network edge, devices not only request services and information from the Cloud but also handle computing tasks including processing, storage, caching, and load balancing on data sent to and from the Cloud <xref target="Shi"/>. This does not preclude end devices from hosting computation themselves, when possible, independently or as part of a distributed edge computing platform.</t>
<t>Several standards developing organization (SDO) and industry forums have provided definitions of edge and fog computing:</t>
<ul spacing="normal">
<li>ISO defines edge computing as a "form of distributed computing in which significant processing and data storage takes place on nodes which are at the edge of the network" <xref target="ISO_TR"/>.</li>
<li>ETSI defines multi-access edge computing as a "system which provides an IT service environment and cloud-computing capabilities at the edge of an access network which contains one or more type of access technology, and in close proximity to its users" <xref target="ETSI_MEC_01"/>.</li>
<li>The Industry IoT Consortium (IIC, now incorporating what was formerly OpenFog) defines fog computing as "a horizontal, system-level architecture that distributes computing, storage, control and networking functions closer to the users along a cloud-to-thing continuum" <xref target="OpenFog"/>.</li>
</ul>
<t>Based on these definitions, we can summarize a general philosophy of edge computing as distributing the required functions close to users and data, while the difference to classic local systems is the usage of management and orchestration features adopted from cloud computing.</t>
<t>Actors from various industries approach edge computing using different terms and reference models although, in practice, these approaches are not incompatible and may integrate with each other:</t>
<ul spacing="normal">
<li>The telecommunication industry tends to use a model where edge computing services are deployed over Network Function Virtualization (NFV) infrastructure, at aggregation points or in proximity to the user equipment (e.g., gNodeBs) <xref target="ETSI_MEC_03"/>.</li>
<li>Enterprise and campus solutions often interpret edge computing as an "edge cloud", that is, a smaller data center directly connected to the local network (often referred to as "on-premise").</li>
<li>The automation industry defines the edge as the connection point between IT and OT (Operational Technology). Hence, edge computing sometimes refers to applying IT solutions to OT problems, such as analytics, more flexible user interfaces, or simply having more computing power than an automation controller.</li>
</ul>
</section>
<section anchor="sec-uc">
<name>Examples of IoT Edge Computing Use Cases</name>
<t>IoT edge computing can be used in home, industry, grid, healthcare, city, transportation, agriculture, and/or educational scenarios. Here, we discuss only a few examples of such use cases, to identify differentiating requirements, providing references to other use cases.</t>
<t><strong>Smart Factory</strong></t>
<t>As part of the 4th industrial revolution, smart factories run real-time processes based on IT technologies, such as artificial intelligence and big data. Even a very small environmental change in a smart factory can lead to a situation in which production efficiency decreases or product quality problems occur. Therefore, simple but time-sensitive processing can be performed at the edge, for example, controlling the temperature and humidity in the factory, or operating machines based on the real-time collection of the operational status of each machine. However, data requiring highly precise analysis, such as machine lifecycle management or accident risk prediction, can be transferred to a central data center for processing.</t>
<t>The use of edge computing in a smart factory can reduce the cost of network and storage resources by reducing the communication load to the central data center or server. It is also possible to improve process efficiency and facility asset productivity through real-time prediction of failures and to reduce the cost of failure through preliminary measures. In the existing manufacturing field, production facilities are manually run according to a program entered in advance; however, edge computing in a smart factory enables tailoring solutions by analyzing data at each production facility and machine level. Digital twins <xref target="Jones"/> of IoT devices have been jointly used with edge computing in industrial IoT scenarios <xref target="Chen"/>.</t>
<t><strong>Smart Grid</strong></t>
<t>In future smart city scenarios, the Smart Grid will be critical in ensuring highly available/efficient energy control in city-wide electricity management. Edge computing is expected to play a significant role in these systems to improve the transmission efficiency of electricity, to react to, and restore power after a disturbance, to reduce operation costs, and to reuse energy effectively, since these operations involve local decision-making. In addition, edge computing can help monitor power generation and power demand, and make local electrical energy storage decisions in smart grid systems.</t>
<t><strong>Smart Agriculture</strong></t>
<t>Smart agriculture integrates information and communication technologies with farming technology. Intelligent farms use IoT technology to measure and analyze parameters, such as the temperature, humidity, sunlight, carbon dioxide, and soil quality, in crop cultivation facilities. Depending on the analysis results, control devices are used to set the environmental parameters to an appropriate state. Remote management is also possible through mobile devices such as smartphones.</t>
<t>In existing farms, simple systems such as management according to temperature and humidity can be easily and inexpensively implemented using IoT technology. Field sensors gather data on field and crop condition. This data is then transmitted to cloud servers that process data and recommend actions. The use of edge computing can reduce the volume of back-and-forth data transmissions significantly, resulting in cost and bandwidth savings. Locally generated data can be processed at the edge, and local computing and analytics can drive local actions. With edge computing, it is easy for farmers to select large amounts of data for processing, and data can be analyzed even in remote areas with poor access conditions. Other applications include enabling dashboarding, for example, to visualize the farm status, as well as enhancing Extended Reality (XR) applications that require edge audio/video processing. As the number of people working on farming has been decreasing over time, increasing automation enabled by edge computing can be a driving force for future smart agriculture.</t>
<t><strong>Smart Construction</strong></t>
<t>Safety is critical at construction sites. Every year, many construction workers lose their lives because of falls, collisions, electric shocks, and other accidents. Therefore, solutions have been developed to improve construction site safety, including the real-time identification of workers, monitoring of equipment location, and predictive accident prevention. To deploy these solutions, many cameras and IoT sensors have been installed on construction sites, to measure noise, vibration, gas concentration, etc. Typically, the data generated from these measurements is collected in on-site gateways and sent to remote cloud servers for storage and analysis. Thus, an inspector can check the information stored on the cloud server to investigate an incident. However, this approach can be expensive because of transmission costs, for example, of video streams over a mobile network connection, and because usage fees of private cloud services.</t>
<t>Using edge computing, data generated at the construction site can be processed and analyzed on an edge server located within or near the site. Only the result of this processing needs to be transferred to a cloud server, thus reducing transmission costs. It is also possible to locally generate warnings to prevent accidents in real-time.</t>
<t><strong>Self-Driving Car</strong></t>
<t>Edge computing plays a crucial role in safety-focused self-driving car systems. With a multitude of sensors, such as high-resolution cameras, radar, LIDAR, sonar sensors, and GPS systems, autonomous vehicles generate vast amounts of real-time data. Local processing utilizing edge computing nodes allows for efficient collection and analysis of this data to monitor vehicle distances and road conditions and respond promptly to unexpected situations. Roadside computing nodes can also be leveraged to offload tasks when necessary, for example, when the local processing capacity of the car is insufficient because of hardware constraints or a large data volume.</t>
<t>For instance, when the car ahead slows, a self-driving car adjusts its speed to maintain a safe distance, or when a roadside signal changes, it adapts its behavior accordingly. In another example, cars equipped with self-parking features utilize local processing to analyze sensor data, determine suitable parking spots, and execute precise parking maneuvers without relying on external processing or connectivity. It is also possible to use in-cabin cameras coupled with local processing to monitor the driver's attention level and detect signs of drowsiness or distraction. The system can issue warnings or implement preventive measures to ensure driver safety.</t>
<t>Edge computing empowers self-driving cars by enabling real-time processing, reducing latency, enhancing data privacy, and optimizing bandwidth usage. By leveraging local processing capabilities, self-driving cars can make rapid decisions, adapt to changing environments, and ensure safer and more efficient autonomous driving experiences.</t>
<t><strong>Digital Twin</strong></t>
<t>A digital twin can simulate different scenarios and predict outcomes based on real-time data collected from the physical environment. This simulation capability empowers proactive maintenance, optimization of operations, and the prediction of potential issues or failures. Decision makers can use digital twins to test and validate different strategies, identify inefficiencies, and optimize performance.</t>
<t>With edge computing, real-time data is collected, processed, and analyzed directly at the edge, allowing for the accurate monitoring and simulation of physical assets. Moreover, edge computing effectively minimizes latency, enabling rapid responses to dynamic conditions as computational resources are brought closer to the physical object. Running digital twin processing at the edge enables organizations to obtain timely insights and make informed decisions that maximize efficiency and performance.</t>
<t><strong>Other Use Cases</strong></t>
<t>AI/ML systems at the edge empower real-time analysis, faster decision-making, reduced latency, improved operational efficiency, and personalized experiences across various industries, by bringing artificial intelligence and machine learning capabilities closer to edge devices.</t>
<t>In addition, oneM2M has studied several IoT edge computing use cases, which are documented in <xref target="oneM2M-TR0001"/>, <xref target="oneM2M-TR0018"/> and <xref target="oneM2M-TR0026"/>. The edge computing related requirements raised through the analysis of these use cases are captured in <xref target="oneM2M-TS0002"/>.</t>
</section>
</section>
<section anchor="sec-challenges">
<name>IoT Challenges Leading Towards Edge Computing</name>
<t>This section describes the challenges faced by IoT that are motivating the adoption of edge computing. These are distinct from the research challenges applicable to IoT edge computing, some of which are mentioned in <xref target="sec-functions"/>.</t>
<t>IoT technology is used with increasingly demanding applications, for example, in industrial, automotive and healthcare domains, leading to new challenges. For example, industrial machines such as laser cutters produce over 1 terabyte of data per hour, and similar amounts can be generated in autonomous cars <xref target="NVIDIA"/>. 90% of IoT data is expected to be stored, processed, analyzed, and acted upon close to the source <xref target="Kelly"/>, as cloud computing models alone cannot address these new challenges <xref target="Chiang"/>.</t>
<t>Below, we discuss IoT use case requirements that are moving cloud capabilities to be more proximate, distributed, and disaggregated.</t>
<section anchor="time-sensitivity">
<name>Time Sensitivity</name>
<t>Many industrial control systems, such as manufacturing systems, smart grids, and oil and gas systems often require stringent end-to-end latency between the sensor and control nodes. While some IoT applications may require latency below a few tens of milliseconds <xref target="Weiner"/>, industrial robots and motion control systems have use cases for cycle times in the order of microseconds <xref target="_60802"/>. In some cases, speed-of-light limitations may simply prevent a cloud-based solutions; however, this is not the only challenge relative to time sensitivity. Guarantees for bounded latency and jitter (<xref target="RFC8578"/> section 7) are also important for industrial IoT applications. This means that control packets must arrive with as little variation as possible and within a strict deadline. Given the best-effort characteristics of the Internet, this challenge is virtually impossible to address, without using end-to-end guarantees for individual message delivery and continuous data flows.</t>
</section>
<section anchor="connectivity-cost">
<name>Connectivity Cost</name>
<t>Some IoT deployments may not face bandwidth constraints when uploading data to the Cloud. 5G and Wi-Fi 6 networks both theoretically top out at 10 gigabits per second (i.e., 4.5 terabytes per hour), allowing to transfer large amounts of uplink data. However, the cost of maintaining continuous high-bandwidth connectivity for such usage is unjustifiable and impractical for most IoT applications. In some settings, for example, in aeronautical communication, higher communication costs reduce the amount of data that can be practically uploaded even further. Minimizing reliance on high-bandwidth connectivity is therefore a requirement, for example, by processing data at the edge and deriving summarized or actionable insights that can be transmitted to the Cloud.</t>
</section>
<section anchor="resilience-to-intermittent-services">
<name>Resilience to Intermittent Services</name>
<t>Many IoT devices, such as sensors, actuators, and controllers, have very limited hardware resources and cannot rely solely on their own resources to meet their computing and/or storage needs. They require reliable, uninterrupted, or resilient services to augment their capabilities to fulfill their application tasks. This is difficult and partly impossible to achieve using cloud services for systems such as vehicles, drones, or oil rigs that have intermittent network connectivity. Conversely, a cloud back-end might want to device data even if it is currently asleep.</t>
</section>
<section anchor="sec-priv">
<name>Privacy and Security</name>
<t>When IoT services are deployed at home, personal information can be learned from detected usage data. For example, one can extract information about employment, family status, age, and income by analyzing smart-meter data <xref target="ENERGY"/>. Policy makers have begun to provide frameworks that limit the usage of personal data and impose strict requirements on data controllers and processors. Data stored indefinitely in the Cloud also increases the risk of data leakage, for instance, through attacks on rich targets.</t>
<t>It is often argues that industrial systems do not provide privacy implications, as no personal data is gathered. However, data from such systems is often highly sensitive, as one might be able to infer trade secrets such as the setup of production lines. Hence, owners of these systems are generally reluctant to upload IoT data to the Cloud.</t>
<t>Furthermore, passive observers can perform traffic analysis on device-to-cloud paths. Therefore, hiding traffic patterns associated with sensor networks can be another requirement for edge computing.</t>
</section>
</section>
<section anchor="sec-functions">
<name>IoT Edge Computing Functions</name>
<t>We first look at the current state of IoT edge computing (<xref target="sec-overview"/>), and then define a general system model (<xref target="sec-model"/>). This provides a context for IoT edge-computing functions, which are listed in <xref target="sec-components-oam"/>, <xref target="sec-components-functional"/> and <xref target="sec-components-app"/>.</t>
<section anchor="sec-overview">
<name>Overview of IoT Edge Computing Today</name>
<t>This section provides an overview of today's IoT edge computing field based on a limited review of standards, research, open-source and proprietary products in <xref target="I-D.defoy-t2trg-iot-edge-computing-background"/>.</t>
<t>IoT gateways, both open-source (such as EdgeX Foundry or Home Edge) and proprietary products, represent a common class of IoT edge-computing products, where the gateway provides a local service on customer premises and is remotely managed through a cloud service. IoT communication protocols are typically used between IoT devices and the gateway, including CoAP <xref target="RFC7252"/>, MQTT <xref target="mqtt5"/>, and many specialized IoT protocols (such as OPC UA and DDS in the Industrial IoT space), while the gateway communicates with the distant cloud typically using HTTPS. Virtualization platforms enable the deployment of virtual edge computing functions (using VMs and application containers), including IoT gateway software, on servers in the mobile network infrastructure (at base stations and concentration points), edge data centers (in central offices), and regional data centers located near central offices. End devices are envisioned to become computing devices in forward-looking projects, but are not commonly used today.</t>
<t>In addition to open-source and proprietary solutions, a horizontal IoT service layer is standardized by the oneM2M standards body to reduce fragmentation, increase interoperability and promote reuse in the IoT ecosystem. Furthermore, ETSI MEC developed an IoT API <xref target="ETSI_MEC_33"/> that enables the deployment of heterogeneous IoT platforms and provides a means to configure the various components of an IoT system.</t>
<t>Physical or virtual IoT gateways can host application programs that are typically built using an SDK to access local services through a programmatic API. Edge cloud system operators host their customers' application VMs or containers on servers located in or near access networks that can implement local edge services. For example, mobile networks can provide edge services for radio-network information, location, and bandwidth management.</t>
<t>Resilience in the IoT can entail the ability to operate autonomously in periods of disconnectedness to preserve the integrity and safety of the controlled system, possibly in a degraded mode. IoT devices and gateways are often expected to operate in always-on and unattended modes, using fault detection and unassisted recovery functions.</t>
<t>The life cycle management of services and applications on physical IoT gateways is generally cloud-based. Edge cloud management platforms and products (such as StarlingX, Akraino Edge Stack, or proprietary products from major Cloud providers) adapt cloud management technologies (e.g., Kubernetes) to the edge cloud, that is, to smaller, distributed computing devices running outside a controlled data center. The service and application life-cycle is typically using an NFV-like management and orchestration model.</t>
<t>The platform typically enables advertising or consuming services hosted on the platform (e.g., the Mp1 interface in ETSI MEC supports service discovery and communication), and enables communication with local and remote endpoints (e.g., message routing function in IoT gateways). The platform is typically extensible to edge applications because it can advertise a service that other edge applications can consume. The IoT communication services include protocol translation, analytics, and transcoding. Communication between edge-computing devices is enabled in tiered or distributed deployments.</t>
<t>An edge cloud platform may enable pass-through without storage or local storage (e.g., on IoT gateways). Some edge cloud platforms use distributed storage such as that provided by a distributed storage platform (e.g., EdgeFS, Ceph), or, in more experimental settings, by an ICN network, for example, systems such as Chipmunk <xref target="chipmunk"/> and Kua <xref target="kua"/> have been proposed as distributed information-centric objects stores. External storage, for example, on databases in distant or local IT cloud, is typically used for filtered data deemed worthy of long-term storage, although in some cases it may be for all data, for example when required for regulatory reasons.</t>
<t>Stateful computing is supported on platforms that host native programs, VMs, or containers. Stateless computing is supported on platforms providing a "serverless computing" service (also known as function-as-a-service, e.g., using stateless containers), or on systems based on named function networking.</t>
<t>In many IoT use cases, a typical network usage pattern is a high volume uplink with some form of traffic reduction enabled by processing over edge-computing devices. Alternatives to traffic reduction include deferred transmission (to off-peak hours or using physical shipping). Downlink traffic includes application control and software updates. Downlink-heavy traffic patterns are not excluded but are more often associated with non-IoT usage (e.g., video CDNs).</t>
</section>
<section anchor="sec-model">
<name>General Model</name>
<t>Edge computing is expected to play an important role in deploying new IoT services integrated with Big Data and AI enabled by flexible in-network computing platforms. Although there are many approaches to edge computing, in this section, we attempt to lay out a general model and the list associated logical functions. In practice, this model can be mapped to different architectures, such as:</t>
<ul spacing="normal">
<li>A single IoT gateway, or a hierarchy of IoT gateways, typically connected to the cloud (e.g., to extend the traditional cloud-based management of IoT devices and data to the edge). The IoT gateway plays a common role in providing access to a heterogeneous set of IoT devices/sensors, handling IoT data, and delivering IoT data to its final destination in a cloud network. Whereas an IoT gateway requires interactions with the cloud, it can also operate independently in a disconnected mode.</li>
<li>A set of distributed computing nodes, for example, embedded in switches, routers, edge cloud servers, or mobile devices. Some IoT devices have sufficient computing capabilities to participate in such distributed systems owing to advances in hardware technology. In this model, edge-computing nodes can collaborate to share resources.</li>
<li>A hybrid system involving both IoT gateways and supporting functions in distributed computing nodes.</li>
</ul>
<t>In the general model described in <xref target="rl-fig1"/>, the edge computing domain is interconnected with IoT devices (southbound connectivity), possibly with a remote/cloud network (northbound connectivity), and with a service operator's system. Edge-computing nodes provide multiple logical functions or components that may not be present in a given system. They may be implemented in a centralized or distributed fashion, at the network edge, or through interworking between the edge network and remote cloud networks.</t>
<figure anchor="rl-fig1">
<name>Model of IoT Edge Computing</name>
<artset>
<artwork type="svg" align="center"><svg xmlns="http://www.w3.org/2000/svg" version="1.1" height="656" width="392" viewBox="0 0 392 656" class="diagram" text-anchor="middle" font-family="monospace" font-size="13px" stroke-linecap="round">
<path d="M 8,128 L 8,528" fill="none" stroke="black"/>
<path d="M 24,576 L 24,624" fill="none" stroke="black"/>
<path d="M 32,32 L 32,80" fill="none" stroke="black"/>
<path d="M 64,528 L 64,576" fill="none" stroke="black"/>
<path d="M 96,576 L 96,624" fill="none" stroke="black"/>
<path d="M 128,80 L 128,128" fill="none" stroke="black"/>
<path d="M 136,576 L 136,624" fill="none" stroke="black"/>
<path d="M 184,528 L 184,576" fill="none" stroke="black"/>
<path d="M 208,32 L 208,80" fill="none" stroke="black"/>
<path d="M 208,576 L 208,624" fill="none" stroke="black"/>
<path d="M 232,48 L 232,96" fill="none" stroke="black"/>
<path d="M 272,560 L 272,624" fill="none" stroke="black"/>
<path d="M 288,96 L 288,128" fill="none" stroke="black"/>
<path d="M 320,528 L 320,560" fill="none" stroke="black"/>
<path d="M 344,560 L 344,624" fill="none" stroke="black"/>
<path d="M 360,48 L 360,96" fill="none" stroke="black"/>
<path d="M 384,128 L 384,544" fill="none" stroke="black"/>
<path d="M 32,32 L 208,32" fill="none" stroke="black"/>
<path d="M 232,48 L 360,48" fill="none" stroke="black"/>
<path d="M 32,80 L 208,80" fill="none" stroke="black"/>
<path d="M 232,96 L 360,96" fill="none" stroke="black"/>
<path d="M 8,128 L 384,128" fill="none" stroke="black"/>
<path d="M 8,528 L 248,528" fill="none" stroke="black"/>
<path d="M 312,528 L 328,528" fill="none" stroke="black"/>
<path d="M 272,560 L 344,560" fill="none" stroke="black"/>
<path d="M 24,576 L 96,576" fill="none" stroke="black"/>
<path d="M 136,576 L 208,576" fill="none" stroke="black"/>
<path d="M 24,624 L 96,624" fill="none" stroke="black"/>
<path d="M 136,624 L 208,624" fill="none" stroke="black"/>
<path d="M 272,624 L 344,624" fill="none" stroke="black"/>
<g class="text">
<text x="84" y="52">Remote</text>
<text x="144" y="52">network</text>
<text x="60" y="68">(e.g.,</text>
<text x="112" y="68">cloud</text>
<text x="172" y="68">network)</text>
<text x="288" y="68">Service</text>
<text x="292" y="84">Operator</text>
<text x="124" y="148">Edge</text>
<text x="184" y="148">Computing</text>
<text x="252" y="148">Domain</text>
<text x="48" y="180">One</text>
<text x="76" y="180">or</text>
<text x="108" y="180">more</text>
<text x="168" y="180">Computing</text>
<text x="232" y="180">Nodes</text>
<text x="52" y="196">(IoT</text>
<text x="108" y="196">gateway,</text>
<text x="160" y="196">end</text>
<text x="212" y="196">devices,</text>
<text x="288" y="196">switches,</text>
<text x="68" y="212">routers,</text>
<text x="168" y="212">mini/micro-data</text>
<text x="268" y="212">centers,</text>
<text x="328" y="212">etc.)</text>
<text x="48" y="244">OAM</text>
<text x="108" y="244">Components</text>
<text x="40" y="260">-</text>
<text x="84" y="260">Resource</text>
<text x="160" y="260">Discovery</text>
<text x="216" y="260">and</text>
<text x="292" y="260">Authentication</text>
<text x="40" y="276">-</text>
<text x="68" y="276">Edge</text>
<text x="140" y="276">Organization</text>
<text x="208" y="276">and</text>
<text x="268" y="276">Federation</text>
<text x="40" y="292">-</text>
<text x="104" y="292">Multi-Tenancy</text>
<text x="176" y="292">and</text>
<text x="232" y="292">Isolation</text>
<text x="40" y="308">-</text>
<text x="64" y="308">...</text>
<text x="76" y="340">Functional</text>
<text x="164" y="340">Components</text>
<text x="40" y="356">-</text>
<text x="92" y="356">In-Network</text>
<text x="184" y="356">Computation</text>
<text x="40" y="372">-</text>
<text x="68" y="372">Edge</text>
<text x="120" y="372">Caching</text>
<text x="40" y="388">-</text>
<text x="104" y="388">Communication</text>
<text x="40" y="404">-</text>
<text x="72" y="404">Other</text>
<text x="132" y="404">Services</text>
<text x="40" y="420">-</text>
<text x="64" y="420">...</text>
<text x="80" y="452">Application</text>
<text x="172" y="452">Components</text>
<text x="40" y="468">-</text>
<text x="64" y="468">IoT</text>
<text x="112" y="468">Devices</text>
<text x="188" y="468">Management</text>
<text x="40" y="484">-</text>
<text x="68" y="484">Data</text>
<text x="132" y="484">Management</text>
<text x="192" y="484">and</text>
<text x="248" y="484">Analytics</text>
<text x="40" y="500">-</text>
<text x="64" y="500">...</text>
<text x="264" y="532">-</text>
<text x="280" y="532">-</text>
<text x="296" y="532">-</text>
<text x="344" y="532">-</text>
<text x="360" y="532">-</text>
<text x="376" y="532">-</text>
<text x="248" y="548">|</text>
<text x="248" y="580">|</text>
<text x="304" y="580">compute</text>
<text x="384" y="580">|</text>
<text x="56" y="596">End</text>
<text x="168" y="596">End</text>
<text x="232" y="596">...</text>
<text x="308" y="596">node/end</text>
<text x="52" y="612">Device</text>
<text x="88" y="612">1</text>
<text x="164" y="612">Device</text>
<text x="200" y="612">2</text>
<text x="236" y="612">...|</text>
<text x="300" y="612">device</text>
<text x="336" y="612">n</text>
<text x="384" y="612">|</text>
<text x="248" y="644">+</text>
<text x="264" y="644">-</text>
<text x="280" y="644">-</text>
<text x="296" y="644">-</text>
<text x="312" y="644">-</text>
<text x="328" y="644">-</text>
<text x="344" y="644">-</text>
<text x="360" y="644">-</text>
<text x="380" y="644">-+</text>
</g>
</svg>
</artwork>
<artwork type="ascii-art" align="center"><![CDATA[
+---------------------+
| Remote network | +---------------+
|(e.g., cloud network)| | Service |
+-----------+---------+ | Operator |
| +------+--------+
| |
+--------------+-------------------+-----------+
| Edge Computing Domain |
| |
| One or more Computing Nodes |
| (IoT gateway, end devices, switches, |
| routers, mini/micro-data centers, etc.) |
| |
| OAM Components |
| - Resource Discovery and Authentication |
| - Edge Organization and Federation |
| - Multi-Tenancy and Isolation |
| - ... |
| |
| Functional Components |
| - In-Network Computation |
| - Edge Caching |
| - Communication |
| - Other Services |
| - ... |
| |
| Application Components |
| - IoT Devices Management |
| - Data Management and Analytics |
| - ... |
| |
+------+--------------+-------- - - - -+- - - -+
| | | | |
| | +-----+--+
+----+---+ +-----+--+ | |compute | |
| End | | End | ... |node/end|
|Device 1| |Device 2| ...| |device n| |
+--------+ +--------+ +--------+
+ - - - - - - - -+
]]></artwork>
</artset>
</figure>
<t>In the distributed model described in <xref target="rl-fig2"/>, the edge-computing domain is composed of IoT edge gateways and IoT devices which are also used as computing nodes. Edge computing domains are connected to a remote/cloud network and their respective service operator's system. IoT devices/computing nodes provide logical functions, for example as part of distributed machine learning or distributed image processing applications. The processing capabilities in IoT devices are limited; they require the support of other nodes, and in a distributed machine learning application, the training process for AI services can be executed at IoT edge gateways or cloud networks and the prediction (inference) service is executed in the IoT devices. In a distributed image processing application, some image processing functions can be similarly executed at the edge or in the cloud, while preprocessing, which helps limiting the amount of uploaded data, is performed by the IoT device.</t>
<figure anchor="rl-fig2">
<name>Example: Machine Learning over a Distributed IoT Edge Computing System</name>
<artset>
<artwork type="svg" align="center"><svg xmlns="http://www.w3.org/2000/svg" version="1.1" height="576" width="392" viewBox="0 0 392 576" class="diagram" text-anchor="middle" font-family="monospace" font-size="13px" stroke-linecap="round">
<path d="M 8,32 L 8,240" fill="none" stroke="black"/>
<path d="M 8,352 L 8,560" fill="none" stroke="black"/>
<path d="M 24,80 L 24,144" fill="none" stroke="black"/>
<path d="M 24,448 L 24,512" fill="none" stroke="black"/>
<path d="M 32,176 L 32,208" fill="none" stroke="black"/>
<path d="M 32,272 L 32,320" fill="none" stroke="black"/>
<path d="M 32,384 L 32,416" fill="none" stroke="black"/>
<path d="M 64,144 L 64,176" fill="none" stroke="black"/>
<path d="M 64,416 L 64,448" fill="none" stroke="black"/>
<path d="M 96,80 L 96,144" fill="none" stroke="black"/>
<path d="M 96,448 L 96,512" fill="none" stroke="black"/>
<path d="M 128,208 L 128,272" fill="none" stroke="black"/>
<path d="M 128,320 L 128,384" fill="none" stroke="black"/>
<path d="M 136,80 L 136,144" fill="none" stroke="black"/>
<path d="M 136,448 L 136,512" fill="none" stroke="black"/>
<path d="M 176,144 L 176,176" fill="none" stroke="black"/>
<path d="M 176,416 L 176,448" fill="none" stroke="black"/>
<path d="M 208,80 L 208,144" fill="none" stroke="black"/>
<path d="M 208,272 L 208,320" fill="none" stroke="black"/>
<path d="M 208,448 L 208,512" fill="none" stroke="black"/>
<path d="M 232,272 L 232,320" fill="none" stroke="black"/>
<path d="M 280,80 L 280,144" fill="none" stroke="black"/>
<path d="M 280,448 L 280,512" fill="none" stroke="black"/>
<path d="M 288,208 L 288,272" fill="none" stroke="black"/>
<path d="M 288,320 L 288,384" fill="none" stroke="black"/>
<path d="M 320,144 L 320,176" fill="none" stroke="black"/>
<path d="M 320,416 L 320,448" fill="none" stroke="black"/>
<path d="M 344,176 L 344,208" fill="none" stroke="black"/>
<path d="M 344,384 L 344,416" fill="none" stroke="black"/>
<path d="M 352,80 L 352,144" fill="none" stroke="black"/>
<path d="M 352,272 L 352,320" fill="none" stroke="black"/>
<path d="M 352,448 L 352,512" fill="none" stroke="black"/>
<path d="M 384,32 L 384,240" fill="none" stroke="black"/>
<path d="M 384,352 L 384,560" fill="none" stroke="black"/>
<path d="M 8,32 L 384,32" fill="none" stroke="black"/>
<path d="M 24,80 L 96,80" fill="none" stroke="black"/>
<path d="M 136,80 L 208,80" fill="none" stroke="black"/>
<path d="M 280,80 L 352,80" fill="none" stroke="black"/>
<path d="M 24,144 L 96,144" fill="none" stroke="black"/>
<path d="M 136,144 L 208,144" fill="none" stroke="black"/>
<path d="M 280,144 L 352,144" fill="none" stroke="black"/>
<path d="M 32,176 L 344,176" fill="none" stroke="black"/>
<path d="M 32,208 L 344,208" fill="none" stroke="black"/>
<path d="M 8,240 L 384,240" fill="none" stroke="black"/>
<path d="M 32,272 L 208,272" fill="none" stroke="black"/>
<path d="M 232,272 L 352,272" fill="none" stroke="black"/>
<path d="M 32,320 L 208,320" fill="none" stroke="black"/>
<path d="M 232,320 L 352,320" fill="none" stroke="black"/>
<path d="M 8,352 L 384,352" fill="none" stroke="black"/>
<path d="M 32,384 L 344,384" fill="none" stroke="black"/>
<path d="M 32,416 L 344,416" fill="none" stroke="black"/>
<path d="M 24,448 L 96,448" fill="none" stroke="black"/>
<path d="M 136,448 L 208,448" fill="none" stroke="black"/>
<path d="M 280,448 L 352,448" fill="none" stroke="black"/>
<path d="M 24,512 L 96,512" fill="none" stroke="black"/>
<path d="M 136,512 L 208,512" fill="none" stroke="black"/>
<path d="M 280,512 L 352,512" fill="none" stroke="black"/>
<path d="M 8,560 L 384,560" fill="none" stroke="black"/>
<g class="text">
<text x="124" y="52">Edge</text>
<text x="184" y="52">Computing</text>
<text x="252" y="52">Domain</text>
<text x="56" y="100">Compute</text>
<text x="168" y="100">Compute</text>
<text x="312" y="100">Compute</text>
<text x="60" y="116">node/End</text>
<text x="172" y="116">node/End</text>
<text x="244" y="116">....</text>
<text x="316" y="116">node/End</text>
<text x="52" y="132">device</text>
<text x="88" y="132">1</text>
<text x="164" y="132">device</text>
<text x="200" y="132">2</text>
<text x="244" y="132">....</text>
<text x="308" y="132">device</text>
<text x="344" y="132">m</text>
<text x="136" y="196">IoT</text>
<text x="172" y="196">Edge</text>
<text x="224" y="196">Gateway</text>
<text x="84" y="292">Remote</text>
<text x="144" y="292">network</text>
<text x="288" y="292">Service</text>
<text x="60" y="308">(e.g.,</text>
<text x="112" y="308">cloud</text>
<text x="172" y="308">network)</text>
<text x="296" y="308">Operator(s)</text>
<text x="136" y="404">IoT</text>
<text x="172" y="404">Edge</text>
<text x="224" y="404">Gateway</text>
<text x="56" y="468">Compute</text>
<text x="168" y="468">Compute</text>
<text x="312" y="468">Compute</text>
<text x="60" y="484">node/End</text>
<text x="172" y="484">node/End</text>
<text x="244" y="484">....</text>
<text x="316" y="484">node/End</text>
<text x="52" y="500">device</text>
<text x="88" y="500">1</text>
<text x="164" y="500">device</text>
<text x="200" y="500">2</text>
<text x="244" y="500">....</text>
<text x="308" y="500">device</text>
<text x="344" y="500">n</text>
<text x="124" y="548">Edge</text>
<text x="184" y="548">Computing</text>
<text x="252" y="548">Domain</text>
</g>
</svg>
</artwork>
<artwork type="ascii-art" align="center"><![CDATA[
+----------------------------------------------+
| Edge Computing Domain |
| |
| +--------+ +--------+ +--------+ |
| |Compute | |Compute | |Compute | |
| |node/End| |node/End| .... |node/End| |
| |device 1| |device 2| .... |device m| |
| +----+---+ +----+---+ +----+---+ |
| | | | |
| +---+-------------+-----------------+--+ |
| | IoT Edge Gateway | |
| +-----------+-------------------+------+ |
| | | |
+--------------+-------------------+-----------+
| |
+-----------+---------+ +------+-------+
| Remote network | | Service |
|(e.g., cloud network)| | Operator(s) |
+-----------+---------+ +------+-------+
| |
+--------------+-------------------+-----------+
| | | |
| +-----------+-------------------+------+ |
| | IoT Edge Gateway | |
| +---+-------------+-----------------+--+ |
| | | | |
| +----+---+ +----+---+ +----+---+ |
| |Compute | |Compute | |Compute | |
| |node/End| |node/End| .... |node/End| |
| |device 1| |device 2| .... |device n| |
| +--------+ +--------+ +--------+ |
| |
| Edge Computing Domain |
+----------------------------------------------+
]]></artwork>
</artset>
</figure>
<t>In the following, we enumerate major edge computing domain components. They are here loosely organized into OAM (Operations, Administration, and Maintenance), functional, and application components, with the understanding that the distinction between these classes may not always be clear, depending on actual system architectures. Some representative research challenges are associated with those functions. We used input from co-authors, IRTF attendees, and some comprehensive reviews of the field (<xref target="Yousefpour"/>, <xref target="Zhang2"/>, <xref target="Khan"/>).</t>
</section>
<section anchor="sec-components-oam">
<name>OAM Components</name>
<t>Edge computing OAM extends beyond the network-related OAM functions listed in <xref target="RFC6291"/>. In addition to infrastructure (network, storage, and computing resources), edge computing systems can also include computing environments (for VMs, software containers, functions), IoT devices, data, and code.</t>
<t>Operation-related functions include performance monitoring for service-level agreement measurements, fault management and provisioning for links, nodes, compute and storage resources, platforms, and services. Administration covers network/compute/storage resources, platforms and services discovery, configuration, and planning. Discovery during normal operation (e.g., discovery of compute or storage nodes by endpoints) is typically not included in OAM; however, in this document, we do not address it separately. Management covers the monitoring and diagnostics of failures, as well as means to minimize their occurrence and take corrective actions. This may include software update management and high service availability through redundancy and multipath communication. Centralized (e.g., SDN) and decentralized management systems can be used. Finally, we arbitrarily chose to address data management as an application component, however, in some systems, data management may be considered similar to a network management function.</t>
<t>We further detail a few relevant OAM components.</t>
<section anchor="sec-dis-auth">
<name>Resource Discovery and Authentication</name>
<t>Discovery and authentication may target platforms and , infrastructure resources, such as computing, networking, and storage, as well as other resources such as IoT devices, sensors, data, code units, services, applications, and users interacting with the system. Broker-based solutions can be used, for example, using an IoT gateway as a broker to discover IoT resources. More decentralized solutions can also be used in replacement or complement, for example, CoAP enables multicast discovery of an IoT device, and CoAP service discovery enables obtaining a list of resources made available by this device <xref target="RFC7252"/>. For device authentication, current centralized gateway-based systems rely on the installation of a secret on IoT devices and computing devices (e.g., a device certificate stored in a hardware security module, or a combination of code and data stored in a trusted execution environment).</t>
<t>Related challenges include:</t>
<ul spacing="normal">
<li>Discovery, authentication, and trust establishment between IoT devices, compute nodes, and platforms, with regard to concerns such as mobility, heterogeneous devices and networks, scale, multiple trust domains, constrained devices, anonymity, and traceability.</li>
<li>Intermittent connectivity to the Internet, removing the need to rely on a third-party authority <xref target="Echeverria"/>.</li>
<li>Resiliency to failure <xref target="Harchol"/>, denial of service attacks, easier physical access for attackers.</li>
</ul>
</section>
<section anchor="edge-organization-and-federation">
<name>Edge Organization and Federation</name>
<t>In a distributed system context, once edge devices have discovered and authenticated each other, they can be organized, or self-organized, into hierarchies or clusters. The organizational structure may range from centralized to peer-to-peer, or it may be closely tied to other systems. Such groups can also form federations with other edges or with remote clouds.</t>
<t>Related challenges include:</t>
<ul spacing="normal">
<li>Support for scaling, and enabling fault-tolerance or self-healing <xref target="Jeong"/>. In addition to using a hierarchical organization to cope with scaling, another available and possibly complementary mechanism is multicast (<xref target="RFC7390"/> <xref target="I-D.ietf-core-groupcomm-bis"/>). Other approaches include relying on blockchains <xref target="Ali"/>.</li>
<li>Integration of edge computing with virtualized Radio Access Networks (Fog RAN) <xref target="I-D.bernardos-sfc-fog-ran"/> and 5G access networks.</li>
<li>Sharing resources in multi-vendor/operator scenarios, to optimize criteria such as profit <xref target="Anglano"/>, resource usage, latency, and energy consumption.</li>
<li>Capacity planning, placement of infrastructure nodes to minimize delay <xref target="Fan"/>, cost, energy, etc.</li>
<li>Incentives for participation, for example, in peer-to-peer federation schemes.</li>
<li>Design of federated AI over IoT edge computing systems <xref target="Brecko"/>, for example, for anomaly detection.</li>
</ul>
</section>
<section anchor="multi-tenancy-and-isolation">
<name>Multi-Tenancy and Isolation</name>
<t>Some IoT edge computing systems make use of virtualized (compute, storage and networking) resources to address the need for secure multi-tenancy at the edge. This leads to "edge clouds" that share properties with remotes clouds and can reuse some of their ecosystems. Virtualization function management is largely covered by ETSI NFV and MEC standards and recommendations. Projects such as <xref target="LFEDGE-EVE"/> further cover virtualization and its management in distributed edge-computing settings.</t>
<t>Related challenges include:</t>
<ul spacing="normal">
<li>Adapting cloud management platforms to the edge, to account for its distributed nature, e.g., using Conflict-free Replicated Data Types (CRDT) <xref target="Jeffery"/>, heterogeneity and customization, e.g., using intent-based management mechanisms <xref target="Cao"/>, and limited resources.</li>
<li>Minimizing virtual function instantiation time and resource usage.</li>
</ul>
</section>
</section>
<section anchor="sec-components-functional">
<name>Functional Components</name>
<section anchor="in-network-computation">
<name>In-Network Computation</name>
<t>A core function of IoT edge computing is to enable local computation on a node at the network edge, typically for application-layer processing, such as processing input data from sensors, making local decisions, preprocessing data, offloading computation on behalf of a device, service, or user. Related functions include orchestrating computation (in a centralized or distributed manner) and managing application lifecycles. Support for in-network computation may vary in terms of capability, for example, computing nodes can host virtual machines, software containers, software actors, uni-kernels running stateful or stateless code, or a rule engine providing an API to register actions in response to conditions such as IoT device ID, sensor values to check, thresholds, etc.</t>
<t>Edge offloading includes offloading to and from an IoT device, and to and from a network node. <xref target="Cloudlets"/> offer an example of offloading computation from an end device to a network node. In contrast, oneM2M is an example of a system that allows a cloud-based IoT platform to transfer resources and tasks to a target edge node <xref target="oneM2M-TR0052"/>. Once transferred, the edge node can directly support IoT devices that it serves with the service offloaded by the cloud (e.g., group management, location management, etc.).</t>
<t>QoS can be provided in some systems through the combination of network QoS (e.g., traffic engineering or wireless resource scheduling) and compute/storage resource allocations. For example, in some systems, a bandwidth manager service can be exposed to enable allocation of the bandwidth to/from an edge-computing application instance.</t>
<t>In-network computation can leverage the underlying services, provided using data generated by IoT devices and access networks. Such services include IoT device location, radio network information, bandwidth management and congestion management (e.g., the congestion management feature of oneM2M <xref target="oneM2M-TR0052"/>).</t>
<t>Related challenges include:</t>
<ul spacing="normal">
<li>(Computation placement) Selecting, in a centralized or distributed/peer-to-peer manner, an appropriate compute device based on available resources, location of data input and data sinks, compute node properties, etc., and with varying goals including end-to-end latency, privacy, high availability, energy conservation, or network efficiency, for example, using load-balancing techniques to avoid congestion.</li>
<li>Onboarding code on a platform or computing device, and invoking remote code execution, possibly as part of a distributed programming model and with respect to similar concerns of latency, privacy, etc.: For example, offloading can be included in a vehicular scenario <xref target="Grewe"/>. These operations should deal with heterogeneous compute nodes <xref target="Schafer"/>, and may also support end devices, including IoT devices, as compute nodes <xref target="Larrea"/>.</li>
<li>Adapting Quality of Results (QoR) for applications where a perfect result is not necessary <xref target="Li"/>.</li>
<li>Assisted or automatic partitioning of code: for example, for application programs <xref target="I-D.sarathchandra-coin-appcentres"/> or network programs <xref target="I-D.hsingh-coinrg-reqs-p4comp"/>.</li>
<li>Supporting computation across trust domains: for example, verifying computation results.</li>
<li>Support for computation mobility: relocating an instance from one compute node to another, while maintaining a given service level; session continuity when communicating with end devices that are mobile, possibly at high speed (e.g., in vehicular scenarios); defining lightweight execution environments for secure code mobility, for example, using WebAssembly <xref target="Nieke"/>.</li>
<li>Defining, managing, and verifying Service Level Agreements (SLA) for edge-computing systems: pricing is a challenging task.</li>
</ul>
</section>
<section anchor="edge-storage-and-caching">
<name>Edge Storage and Caching</name>
<t>Local storage or caching enables local data processing (e.g., preprocessing or analysis) as well as delayed data transfer to the cloud or delayed physical shipping. An edge node may offer local data storage (in which persistence is subject to retention policies), caching, or both. Caching generally refers to temporary storage to improve performance without persistence guarantees. An edge-caching component manages data persistence, for example, it schedules the removal of data when it is no longer needed. Other related aspects include the authentication and encryption of data. Edge storage and caching can take the form of a distributed storage systems.</t>
<t>Related challenges include:</t>
<ul spacing="normal">
<li>(Cache and data placement) Using cache positioning and data placement strategies to minimize data retrieval delay <xref target="Liu"/> and energy consumption. Caches may be positioned in the access network infrastructure or on end devices.</li>
<li>Maintaining consistency, freshness, reliability, and privacy of stored/cached data in systems that are distributed, constrained, and dynamic (e.g., owing to end devices and computing nodes churn or mobility), and which can have additional data governance constraints on data storage location. For example, <xref target="Mortazavi"/> leverages a hierarchical storage organization. Freshness-related metrics include the age of information <xref target="Yates"/> that captures the timeliness of information received from a sender (e.g., an IoT device).</li>
</ul>
</section>
<section anchor="communication">
<name>Communication</name>
<t>An edge cloud may provide a northbound data plane or management plane interface to a remote network, such as a cloud, home or enterprise network. This interface does not exist in stand-alone (local-only) scenarios. To support such an interface when it exists, an edge computing component needs to expose an API, deal with authentication and authorization, and support secure communication.</t>
<t>An edge cloud may provide an API or interface to local or mobile users, for example, to provide access to services and applications, or to manage data published by local/mobile devices.</t>
<t>Edge-computing nodes communicate with IoT devices over a southbound interface, typically for data acquisition and IoT device management.</t>
<t>Communication brokering is a typical function of IoT edge computing that facilitates communication with IoT devices, enabling clients to register as recipients for data from devices, as well as forwarding/routing of traffic to or from IoT devices, enabling various data discovery and redistribution patterns, for example, north-south with clouds, east-west with other edge devices <xref target="I-D.mcbride-edge-data-discovery-overview"/>. Another related aspect is dispatching alerts and notifications to interested consumers both inside and outside the edge-computing domain. Protocol translation, analytics, and video transcoding can also be performed when necessary. Communication brokering may be centralized in some systems, for example, using a hub-and-spoke message broker, or distributed with message buses, possibly in a layered bus approach. Distributed systems can leverage direct communication between end devices over device-to-device links. A broker can ensure communication reliability and traceability and, in some cases, transaction management.</t>
<t>Related challenges include:</t>
<ul spacing="normal">
<li>Defining edge computing abstractions, such as PaaS <xref target="Yangui"/>, suitable for users and cloud systems to interact with edge computing systems and dealing with interoperability issues such as data model heterogeneity.</li>
<li>Enabling secure and resilient communication between IoT devices and remote cloud, for example, through multipath support.</li>
</ul>
</section>
</section>
<section anchor="sec-components-app">
<name>Application Components</name>
<t>IoT edge computing can host applications, such as those mentioned in <xref target="sec-uc"/>. While describing the components of individual applications is out of our scope, some of those applications share similar functions, such as IoT device management and data management, as described below.</t>
<section anchor="iot-device-management">
<name>IoT Device Management</name>
<t>IoT device management includes managing information regarding IoT devices, including their sensors, and how to communicate with them. Edge computing addresses the scalability challenges of a large number of IoT devices by separating the scalability domain into edge/local networks and remote networks. For example, in the context of the oneM2M standard, a device management functionality (called "software campaign" in oneM2M) enables the installation, deletion, activation, and deactivation of software functions/services on a potentially large number of edge nodes <xref target="oneM2M-TR0052"/>. Using a dashboard or management software, a service provider issues these requests through an IoT cloud platform supporting the software campaign functionality.</t>
<t>Challenges listed in <xref target="sec-dis-auth"/> may be applicable to IoT devices management as well.</t>
</section>
<section anchor="sec-data">
<name>Data Management and Analytics</name>
<t>Data storage and processing at the edge are major aspects of IoT edge computing, directly addressing the high-level IoT challenges listed in <xref target="sec-challenges"/>. Data analysis, for example, through AI/ML tasks performed at the edge, may benefit from specialized hardware support on the computing nodes.</t>
<t>Related challenges include:</t>
<ul spacing="normal">
<li>Addressing concerns regarding resource usage, security, and privacy when sharing, processing, discovering, or managing data: for example presenting data in views composed of an aggregation of related data <xref target="Zhang"/>; protecting data communication between authenticated peers <xref target="Basudan"/>, classifying data (e.g., in terms of privacy, importance, validity), and compressing and encrypting data, for example, using homomorphic encryption to directly process encrypted data <xref target="Stanciu"/>.</li>
<li>Other concerns regarding edge data discovery (e.g., streaming data, metadata, and events) include siloization and lack of standards in edge environments that can be dynamic (e.g., vehicular networks) and heterogeneous <xref target="I-D.mcbride-edge-data-discovery-overview"/>.</li>
<li>Data-driven programming models <xref target="Renart"/>, for example, event-based, including handling naming and data abstractions.</li>
<li>Data integration in an environment that without data standardization, or where different sources use different ontologies <xref target="Farnbauer-Schmidt"/>.</li>
<li>Addressing concerns such as limited resources, privacy, dynamic, and heterogeneous environments to deploy machine learning at the edge: for example, making machine learning more lightweight and distributed (e.g., enabling distributed inference at the edge), supporting shorter training times and simplified models, and supporting models that can be compressed for efficient communication <xref target="Murshed"/>.</li>
<li>Although edge computing can support IoT services independently of cloud computing, it can also be connected to cloud computing. Thus, the relationship between IoT edge computing and cloud computing, with regard to data management, is another potential challenge <xref target="ISO_TR"/>.</li>
</ul>
</section>
</section>
<section anchor="simulation-and-emulation-environments">
<name>Simulation and Emulation Environments</name>
<t>IoT Edge Computing introduces new challenges to the simulation and emulation tools used by researchers and developers. A varied set of applications, networks, and computing technologies can coexist in a distributed system, making modeling difficult. Scale, mobility, and resource management are additional challenges <xref target="SimulatingFog"/>.</t>
<t>Tools include simulators, where simplified application logic runs on top of a fog network model, and emulators, where actual applications can be deployed, typically in software containers, over a cloud infrastructure (e.g., Docker and Kubernetes) running over a network emulating network edge conditions such as variable delays, throughput and mobility events. To gain in scale, emulated and simulated systems can be used together in hybrid federation-based approaches <xref target="PseudoDynamicTesting"/>, whereas to gain in realism, physical devices can be interconnected with emulated systems. Examples of related work and platforms include the publicly accessible MEC sandbox work recently initiated in ETSI <xref target="ETSI_Sandbox"/>, and open source simulators and emulators (<xref target="AdvantEDGE"/> emulator and tools cited in <xref target="SimulatingFog"/>). EdgeNet <xref target="Senel"/> is a globally distributed edge cloud for Internet researchers, using nodes contributed by institutions, and based on Docker for containerization and Kubernetes for deployment and node management.</t>
<t>Digital twins are virtual instances of a physical system (twin) that are continually updated with the latter's performance, maintenance, and health status data throughout the life cycle of the physical system. <xref target="Madni"/>. In contrast to a traditional emulation or simulated environment, digital twins, once generated, are maintained in sync by their physical twin, which can be, among many other instances, an IoT device, edge device, an edge network. The benefits of digital twins go beyond those of emulation and include accelerated business processes, enhanced productivity, and faster innovation with reduced costs <xref target="I-D.irtf-nmrg-network-digital-twin-arch"/>.</t>
</section>
</section>
<section anchor="security-considerations">
<name>Security Considerations</name>
<t>Privacy and security are drivers of the adoption of edge computing for the IoT (<xref target="sec-priv"/>). As discussed in <xref target="sec-dis-auth"/>, authentication and trust (among computing nodes, management nodes, and end devices) can be challenging as scale, mobility, and heterogeneity increase. The sometimes disconnected nature of edge resources can avoid reliance on third-party authorities. Distributed edge computing is exposed reliability and denial of service attacks. Personal or proprietary IoT data leakage is also a major threat, particularly because of the distributed nature of the systems (<xref target="sec-data"/>). Furthermore, blockchain-based distributed IoT edge computing must be designed for privacy, since public blockchain addressing does not guarantee absolute anonymity <xref target="Ali"/>.</t>
<t>However, edge computing also offers solutions in the security space: maintaining privacy by computing sensitive data closer to data generators is a major use case for IoT edge computing. An edge cloud can be used to perform actions based on sensitive data or to anonymize or aggregate data prior to transmission to a remote cloud server. Edge computing communication brokering functions can also be used to secure communication between edge and cloud networks.</t>
</section>
<section anchor="conclusion">
<name>Conclusion</name>
<t>IoT edge computing plays an essential role, complementary to the cloud, in enabling IoT systems in certain situations. In this document, we presented use cases and listing the core challenges faced by IoT that drive the need for IoT edge computing. The first part of this document may therefore help focus future research efforts on the aspects of IoT edge computing where it is most useful. The second part of this document presents a general system model and structured overview of the associated research challenges and related work. The structure, based on the system model, is not meant to be restrictive and exists for the purpose of having a link between individual research areas and where they are applicable in an IoT edge computing system.</t>
</section>
<section anchor="iana-considerations">
<name>IANA Considerations</name>
<t>This document has no IANA actions.</t>
</section>
<section anchor="acknowledgements">
<name>Acknowledgements</name>
<t>The authors would like to thank Joo-Sang Youn, Akbar Rahman, Michel Roy, Robert Gazda, Rute Sofia, Thomas Fossati, Chonggang Wang, <contact fullname="Marie-José Montpetit"/>, Carlos J. Bernardos, Milan Milenkovic, Dale Seed, JaeSeung Song, Roberto Morabito, Carsten Bormann and <contact fullname="Ari Keränen"/> for their valuable comments and suggestions on this document.</t>
</section>
</middle>
<back>
<references>
<name>Informative References</name>
<reference anchor="I-D.mcbride-edge-data-discovery-overview">
<front>
<title>Edge Data Discovery for COIN</title>
<author fullname="Mike McBride" initials="M." surname="McBride">
<organization>Futurewei</organization>
</author>
<author fullname="Dirk Kutscher" initials="D." surname="Kutscher">
<organization>Emden University</organization>
</author>
<author fullname="Eve Schooler" initials="E." surname="Schooler">
<organization>Intel</organization>
</author>
<author fullname="Carlos J. Bernardos" initials="C. J." surname="Bernardos">
<organization>Universidad Carlos III de Madrid</organization>
</author>
<author fullname="Diego Lopez" initials="D." surname="Lopez">
<organization>Telefonica</organization>
</author>
<author fullname="Xavier de Foy" initials="X." surname="de Foy">
<organization>InterDigital Communications, LLC</organization>
</author>
<date day="1" month="November" year="2020"/>
<abstract>
<t> This document describes the problem of distributed data discovery in
edge computing, and in particular for computing-in-the-network
(COIN), which may require both the marshalling of data at the outset
of a computation and the persistence of the resultant data after the
computation. Although the data might originate at the network edge,
as more and more distributed data is created, processed, and stored,
it becomes increasingly dispersed throughout the network. There
needs to be a standard way to find it. New and existing protocols
will need to be developed to support distributed data discovery at
the network edge and beyond.
</t>
</abstract>
</front>
<seriesInfo name="Internet-Draft" value="draft-mcbride-edge-data-discovery-overview-05"/>
</reference>
<reference anchor="RFC6291">
<front>
<title>Guidelines for the Use of the "OAM" Acronym in the IETF</title>
<author fullname="L. Andersson" initials="L." surname="Andersson"/>
<author fullname="H. van Helvoort" initials="H." surname="van Helvoort"/>
<author fullname="R. Bonica" initials="R." surname="Bonica"/>
<author fullname="D. Romascanu" initials="D." surname="Romascanu"/>
<author fullname="S. Mansfield" initials="S." surname="Mansfield"/>
<date month="June" year="2011"/>
<abstract>
<t>At first glance, the acronym "OAM" seems to be well-known and well-understood. Looking at the acronym a bit more closely reveals a set of recurring problems that are revisited time and again.</t>
<t>This document provides a definition of the acronym "OAM" (Operations, Administration, and Maintenance) for use in all future IETF documents that refer to OAM. There are other definitions and acronyms that will be discussed while exploring the definition of the constituent parts of the "OAM" term. This memo documents an Internet Best Current Practice.</t>
</abstract>
</front>
<seriesInfo name="BCP" value="161"/>
<seriesInfo name="RFC" value="6291"/>
<seriesInfo name="DOI" value="10.17487/RFC6291"/>
</reference>
<reference anchor="RFC8578">
<front>
<title>Deterministic Networking Use Cases</title>
<author fullname="E. Grossman" initials="E." role="editor" surname="Grossman"/>
<date month="May" year="2019"/>
<abstract>
<t>This document presents use cases for diverse industries that have in common a need for "deterministic flows". "Deterministic" in this context means that such flows provide guaranteed bandwidth, bounded latency, and other properties germane to the transport of time-sensitive data. These use cases differ notably in their network topologies and specific desired behavior, providing as a group broad industry context for Deterministic Networking (DetNet). For each use case, this document will identify the use case, identify representative solutions used today, and describe potential improvements that DetNet can enable.</t>
</abstract>
</front>
<seriesInfo name="RFC" value="8578"/>
<seriesInfo name="DOI" value="10.17487/RFC8578"/>
</reference>
<reference anchor="I-D.irtf-t2trg-rest-iot">
<front>
<title>Guidance on RESTful Design for Internet of Things Systems</title>
<author fullname="Ari Keränen" initials="A." surname="Keränen">
<organization>Ericsson</organization>
</author>
<author fullname="Matthias Kovatsch" initials="M." surname="Kovatsch">
<organization>Siemens</organization>
</author>
<author fullname="Klaus Hartke" initials="K." surname="Hartke">
</author>
<date day="25" month="July" year="2023"/>
<abstract>
<t> This document gives guidance for designing Internet of Things (IoT)
systems that follow the principles of the Representational State
Transfer (REST) architectural style. This document is a product of
the IRTF Thing-to-Thing Research Group (T2TRG).
</t>
</abstract>
</front>
<seriesInfo name="Internet-Draft" value="draft-irtf-t2trg-rest-iot-12"/>
</reference>
<reference anchor="Ashton" target="http://www.itrco.jp/libraries/RFIDjournal-That%20Internet%20of%20Things%20Thing.pdf">
<front>
<title>That Internet of Things thing</title>
<author initials="K." surname="Ashton">
<organization/>
</author>
<date year="2009"/>
</front>
<refcontent>RFID J. vol. 22, no. 7, pp. 97-114</refcontent>
</reference>
<reference anchor="Lin">
<front>
<title>A Survey on Internet of Things: Architecture, Enabling Technologies, Security and Privacy, and Applications</title>
<author initials="J." surname="Lin" fullname="Jie Lin">
<organization/>
</author>
<author initials="W." surname="Yu" fullname="Wei Yu">
<organization/>
</author>
<author initials="N." surname="Zhang" fullname="Nan Zhang">
<organization/>
</author>
<author initials="X." surname="Yang" fullname="Xinyu Yang">
<organization/>
</author>
<author initials="H." surname="Zhang" fullname="Hanlin Zhang">
<organization/>
</author>
<author initials="W." surname="Zhao" fullname="Wei Zhao">
<organization/>
</author>
<date year="2017" month="October"/>
</front>
<seriesInfo name="IEEE Internet of Things Journal" value="vol. 4, no. 5, pp. 1125-1142"/>
<seriesInfo name="DOI" value="10.1109/jiot.2017.2683200"/>
</reference>
<reference anchor="NIST">
<front>
<title>The NIST definition of cloud computing</title>
<author initials="P." surname="Mell" fullname="P M Mell">
<organization/>
</author>
<author initials="T." surname="Grance" fullname="T Grance">
<organization/>
</author>
<date year="2011"/>
</front>
<seriesInfo name="National Institute of Standards and Technology" value="report"/>
<seriesInfo name="DOI" value="10.6028/nist.sp.800-145"/>
</reference>
<reference anchor="Botta">
<front>
<title>Integration of Cloud computing and Internet of Things: A survey</title>
<author initials="A." surname="Botta" fullname="Alessio Botta">
<organization/>
</author>
<author initials="W." surname="de Donato" fullname="Walter de Donato">
<organization/>
</author>
<author initials="V." surname="Persico" fullname="Valerio Persico">
<organization/>
</author>
<author initials="A." surname="Pescape" fullname="Antonio Pescape">
<organization/>
</author>
<date year="2016" month="March"/>
</front>
<seriesInfo name="Future Generation Computer Systems" value="vol. 56, pp. 684-700"/>
<seriesInfo name="DOI" value="10.1016/j.future.2015.09.021"/>
</reference>
<reference anchor="Shi">
<front>
<title>Edge Computing: Vision and Challenges</title>
<author initials="W." surname="Shi" fullname="Weisong Shi">
<organization/>
</author>
<author initials="J." surname="Cao" fullname="Jie Cao">
<organization/>
</author>
<author initials="Q." surname="Zhang" fullname="Quan Zhang">
<organization/>
</author>
<author initials="Y." surname="Li" fullname="Youhuizi Li">
<organization/>
</author>
<author initials="L." surname="Xu" fullname="Lanyu Xu">
<organization/>
</author>
<date year="2016" month="October"/>
</front>
<seriesInfo name="IEEE Internet of Things Journal" value="vol. 3, no. 5, pp. 637-646"/>
<seriesInfo name="DOI" value="10.1109/jiot.2016.2579198"/>
</reference>
<reference anchor="Mahadev">
<front>
<title>The Emergence of Edge Computing</title>
<author initials="M." surname="Satyanarayanan" fullname="Mahadev Satyanarayanan">
<organization/>
</author>
<date year="2017" month="January"/>
</front>
<seriesInfo name="Computer" value="vol. 50, no. 1, pp. 30-39"/>
<seriesInfo name="DOI" value="10.1109/mc.2017.9"/>
</reference>
<reference anchor="Chiang">
<front>
<title>Fog and IoT: An Overview of Research Opportunities</title>
<author initials="M." surname="Chiang" fullname="Mung Chiang">
<organization/>
</author>
<author initials="T." surname="Zhang" fullname="Tao Zhang">
<organization/>
</author>
<date year="2016" month="December"/>
</front>
<seriesInfo name="IEEE Internet of Things Journal" value="vol. 3, no. 6, pp. 854-864"/>
<seriesInfo name="DOI" value="10.1109/jiot.2016.2584538"/>
</reference>
<reference anchor="Weiner">
<front>
<title>Design of a low-latency, high-reliability wireless communication system for control applications</title>
<author initials="M." surname="Weiner" fullname="Matthew Weiner">
<organization/>
</author>
<author initials="M." surname="Jorgovanovic" fullname="Milos Jorgovanovic">
<organization/>
</author>
<author initials="A." surname="Sahai" fullname="Anant Sahai">
<organization/>
</author>
<author initials="B." surname="Nikolie" fullname="Borivoje Nikolie">
<organization/>
</author>
<date year="2014" month="June"/>
</front>
<seriesInfo name="2014 IEEE International Conference on Communications" value="(ICC)"/>
<seriesInfo name="DOI" value="10.1109/icc.2014.6883918"/>
</reference>
<reference anchor="Kelly" target="https://campustechnology.com/articles/2015/04/15/internet-of-things-data-to-top-1-6-zettabytes-by-2020.aspx">
<front>
<title>Internet of Things Data to Top 1.6 Zettabytes by 2022</title>
<author initials="R." surname="Kelly">
<organization/>
</author>
<date year="2015"/>
</front>
<refcontent>Retrieved on 2022-05-24</refcontent>
</reference>
<reference anchor="ISO_TR" target="https://www.iso.org/standard/53284.html">
<front>
<title>Internet of things (IoT) - Edge computing</title>
<author>
<organization/>
</author>
<date year="2020"/>
</front>
<seriesInfo name="ISO/IEC" value="TR 30164"/>
</reference>
<reference anchor="OpenFog" target="https://iiconsortium.org/pdf/OpenFog_Reference_Architecture_2_09_17.pdf">
<front>
<title>OpenFog Reference Architecture for Fog Computing</title>
<author>
<organization/>
</author>
<date year="2017"/>
</front>
<refcontent>OpenFog Consortium</refcontent>
</reference>
<reference anchor="ETSI_MEC_03" target="https://www.etsi.org/deliver/etsi_gs/MEC/001_099/003/02.01.01_60/gs_MEC003v020101p.pdf">
<front>
<title>Mobile Edge Computing (MEC); Framework and Reference Architecture</title>
<author>
<organization>ETSI</organization>
</author>
<date year="2019"/>
</front>
<refcontent>ETSI GS MEC 003</refcontent>
</reference>
<reference anchor="ETSI_MEC_01" target="https://www.etsi.org/deliver/etsi_gs/MEC/001_099/001/02.01.01_60/gs_MEC001v020101p.pdf">
<front>
<title>Multi-access Edge Computing (MEC); Terminology</title>
<author initials="" surname="ETSI">
<organization/>
</author>
<date year="2019"/>
</front>
<refcontent>ETSI GS MEC 001</refcontent>
</reference>
<reference anchor="NVIDIA" target="https://devblogs.nvidia.com/training-self-driving-vehicles-challenge-scale/">
<front>
<title>Training AI for Self-Driving Vehicles: the Challenge of Scale</title>
<author initials="A." surname="Grzywaczewski">
<organization/>
</author>
<date year="2017"/>
</front>
<refcontent>NVIDIA Developer Blog, retrieved on 2022-05-24</refcontent>
</reference>
<reference anchor="_60802" target="https://grouper.ieee.org/groups/802/1/files/public/docs2018/60802-industrial-use-cases-0918-v13.pdf">
<front>
<title>Use Cases IEC/IEEE 60802 V1.3</title>
<author>
<organization>IEC/IEEE</organization>
</author>
<date year="2018"/>
</front>
<seriesInfo name="IEC/IEEE" value="60802"/>
</reference>
<reference anchor="ENERGY">
<front>
<title>Revealing household characteristics from smart meter data</title>
<author initials="C." surname="Beckel" fullname="Christian Beckel">
<organization/>
</author>
<author initials="L." surname="Sadamori" fullname="Leyna Sadamori">
<organization/>
</author>
<author initials="T." surname="Staake" fullname="Thorsten Staake">
<organization/>
</author>
<author initials="S." surname="Santini" fullname="Silvia Santini">
<organization/>
</author>
<date year="2014" month="December"/>
</front>
<seriesInfo name="Energy" value="vol. 78, pp. 397-410"/>
<seriesInfo name="DOI" value="10.1016/j.energy.2014.10.025"/>
</reference>
<reference anchor="ETSI_Sandbox" target="https://portal.etsi.org/webapp/WorkProgram/Report_WorkItem.asp?WKI_ID=57671">