mirror of
https://github.com/rtic-rs/rtic.git
synced 2024-12-25 11:29:33 +01:00
319 lines
24 KiB
HTML
319 lines
24 KiB
HTML
<!DOCTYPE HTML>
|
|
<html lang="en" class="light sidebar-visible" dir="ltr">
|
|
<head>
|
|
<!-- Book generated using mdBook -->
|
|
<meta charset="UTF-8">
|
|
<title>Preface - Real-Time Interrupt-driven Concurrency</title>
|
|
|
|
|
|
<!-- Custom HTML head -->
|
|
|
|
<meta name="description" content="">
|
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
<meta name="theme-color" content="#ffffff">
|
|
|
|
<link rel="icon" href="favicon.svg">
|
|
<link rel="shortcut icon" href="favicon.png">
|
|
<link rel="stylesheet" href="css/variables.css">
|
|
<link rel="stylesheet" href="css/general.css">
|
|
<link rel="stylesheet" href="css/chrome.css">
|
|
<link rel="stylesheet" href="css/print.css" media="print">
|
|
|
|
<!-- Fonts -->
|
|
<link rel="stylesheet" href="FontAwesome/css/font-awesome.css">
|
|
<link rel="stylesheet" href="fonts/fonts.css">
|
|
|
|
<!-- Highlight.js Stylesheets -->
|
|
<link rel="stylesheet" href="highlight.css">
|
|
<link rel="stylesheet" href="tomorrow-night.css">
|
|
<link rel="stylesheet" href="ayu-highlight.css">
|
|
|
|
<!-- Custom theme stylesheets -->
|
|
|
|
|
|
<!-- Provide site root to javascript -->
|
|
<script>
|
|
var path_to_root = "";
|
|
var default_theme = window.matchMedia("(prefers-color-scheme: dark)").matches ? "navy" : "light";
|
|
</script>
|
|
<!-- Start loading toc.js asap -->
|
|
<script src="toc.js"></script>
|
|
</head>
|
|
<body>
|
|
<div id="body-container">
|
|
<!-- Work around some values being stored in localStorage wrapped in quotes -->
|
|
<script>
|
|
try {
|
|
var theme = localStorage.getItem('mdbook-theme');
|
|
var sidebar = localStorage.getItem('mdbook-sidebar');
|
|
|
|
if (theme.startsWith('"') && theme.endsWith('"')) {
|
|
localStorage.setItem('mdbook-theme', theme.slice(1, theme.length - 1));
|
|
}
|
|
|
|
if (sidebar.startsWith('"') && sidebar.endsWith('"')) {
|
|
localStorage.setItem('mdbook-sidebar', sidebar.slice(1, sidebar.length - 1));
|
|
}
|
|
} catch (e) { }
|
|
</script>
|
|
|
|
<!-- Set the theme before any content is loaded, prevents flash -->
|
|
<script>
|
|
var theme;
|
|
try { theme = localStorage.getItem('mdbook-theme'); } catch(e) { }
|
|
if (theme === null || theme === undefined) { theme = default_theme; }
|
|
const html = document.documentElement;
|
|
html.classList.remove('light')
|
|
html.classList.add(theme);
|
|
html.classList.add("js");
|
|
</script>
|
|
|
|
<input type="checkbox" id="sidebar-toggle-anchor" class="hidden">
|
|
|
|
<!-- Hide / unhide sidebar before it is displayed -->
|
|
<script>
|
|
var sidebar = null;
|
|
var sidebar_toggle = document.getElementById("sidebar-toggle-anchor");
|
|
if (document.body.clientWidth >= 1080) {
|
|
try { sidebar = localStorage.getItem('mdbook-sidebar'); } catch(e) { }
|
|
sidebar = sidebar || 'visible';
|
|
} else {
|
|
sidebar = 'hidden';
|
|
}
|
|
sidebar_toggle.checked = sidebar === 'visible';
|
|
html.classList.remove('sidebar-visible');
|
|
html.classList.add("sidebar-" + sidebar);
|
|
</script>
|
|
|
|
<nav id="sidebar" class="sidebar" aria-label="Table of contents">
|
|
<!-- populated by js -->
|
|
<mdbook-sidebar-scrollbox class="sidebar-scrollbox"></mdbook-sidebar-scrollbox>
|
|
<noscript>
|
|
<iframe class="sidebar-iframe-outer" src="toc.html"></iframe>
|
|
</noscript>
|
|
<div id="sidebar-resize-handle" class="sidebar-resize-handle">
|
|
<div class="sidebar-resize-indicator"></div>
|
|
</div>
|
|
</nav>
|
|
|
|
<div id="page-wrapper" class="page-wrapper">
|
|
|
|
<div class="page">
|
|
<div id="menu-bar-hover-placeholder"></div>
|
|
<div id="menu-bar" class="menu-bar sticky">
|
|
<div class="left-buttons">
|
|
<label id="sidebar-toggle" class="icon-button" for="sidebar-toggle-anchor" title="Toggle Table of Contents" aria-label="Toggle Table of Contents" aria-controls="sidebar">
|
|
<i class="fa fa-bars"></i>
|
|
</label>
|
|
<button id="theme-toggle" class="icon-button" type="button" title="Change theme" aria-label="Change theme" aria-haspopup="true" aria-expanded="false" aria-controls="theme-list">
|
|
<i class="fa fa-paint-brush"></i>
|
|
</button>
|
|
<ul id="theme-list" class="theme-popup" aria-label="Themes" role="menu">
|
|
<li role="none"><button role="menuitem" class="theme" id="light">Light</button></li>
|
|
<li role="none"><button role="menuitem" class="theme" id="rust">Rust</button></li>
|
|
<li role="none"><button role="menuitem" class="theme" id="coal">Coal</button></li>
|
|
<li role="none"><button role="menuitem" class="theme" id="navy">Navy</button></li>
|
|
<li role="none"><button role="menuitem" class="theme" id="ayu">Ayu</button></li>
|
|
</ul>
|
|
<button id="search-toggle" class="icon-button" type="button" title="Search. (Shortkey: s)" aria-label="Toggle Searchbar" aria-expanded="false" aria-keyshortcuts="S" aria-controls="searchbar">
|
|
<i class="fa fa-search"></i>
|
|
</button>
|
|
</div>
|
|
|
|
<h1 class="menu-title">Real-Time Interrupt-driven Concurrency</h1>
|
|
|
|
<div class="right-buttons">
|
|
<a href="print.html" title="Print this book" aria-label="Print this book">
|
|
<i id="print-button" class="fa fa-print"></i>
|
|
</a>
|
|
<a href="https://github.com/rtic-rs/rtic" title="Git repository" aria-label="Git repository">
|
|
<i id="git-repository-button" class="fa fa-github"></i>
|
|
</a>
|
|
|
|
</div>
|
|
</div>
|
|
|
|
<div id="search-wrapper" class="hidden">
|
|
<form id="searchbar-outer" class="searchbar-outer">
|
|
<input type="search" id="searchbar" name="searchbar" placeholder="Search this book ..." aria-controls="searchresults-outer" aria-describedby="searchresults-header">
|
|
</form>
|
|
<div id="searchresults-outer" class="searchresults-outer hidden">
|
|
<div id="searchresults-header" class="searchresults-header"></div>
|
|
<ul id="searchresults">
|
|
</ul>
|
|
</div>
|
|
</div>
|
|
|
|
<!-- Apply ARIA attributes after the sidebar and the sidebar toggle button are added to the DOM -->
|
|
<script>
|
|
document.getElementById('sidebar-toggle').setAttribute('aria-expanded', sidebar === 'visible');
|
|
document.getElementById('sidebar').setAttribute('aria-hidden', sidebar !== 'visible');
|
|
Array.from(document.querySelectorAll('#sidebar a')).forEach(function(link) {
|
|
link.setAttribute('tabIndex', sidebar === 'visible' ? 0 : -1);
|
|
});
|
|
</script>
|
|
|
|
<div id="content" class="content">
|
|
<main>
|
|
<div align="center"><img width="300" height="300" src="RTIC.svg"></div>
|
|
<div style="font-size: 6em; font-weight: bolder;" align="center">RTIC</div>
|
|
<h1 align="center">The hardware accelerated Rust RTOS</h1>
|
|
<p align="center">A concurrency framework for building real-time systems</p>
|
|
<h1 id="preface"><a class="header" href="#preface">Preface</a></h1>
|
|
<p>This book contains user level documentation for the Real-Time Interrupt-driven Concurrency
|
|
(RTIC) framework. The API reference is available <a href="../../api/">here</a>.</p>
|
|
<p>This is the documentation for RTIC v2.x.</p>
|
|
<p>Older releases:
|
|
<a href="/1">RTIC v1.x</a> | <a href="https://github.com/rtic-rs/rtic/tree/release/v0.5">RTIC v0.5.x (unsupported)</a> | <a href="https://github.com/rtic-rs/rtic/tree/release/v0.4">RTFM v0.4.x (unsupported)</a></p>
|
|
<p><a href="https://crates.io/crates/rtic"><img src="https://img.shields.io/crates/v/rtic" alt="crates.io" /></a>
|
|
<a href="https://docs.rs/rtic"><img src="https://docs.rs/rtic/badge.svg" alt="docs.rs" /></a>
|
|
<a href="https://rtic.rs/"><img src="https://img.shields.io/badge/web-rtic.rs-red.svg?style=flat&label=book&colorB=d33847" alt="book" /></a>
|
|
<a href="https://matrix.to/#/#rtic:matrix.org"><img src="https://img.shields.io/matrix/rtic:matrix.org" alt="matrix" /></a>
|
|
<a href="https://hackmd.io/@xmis9JvZT8Gvo9lOEKyZ4Q/SkBJKsjuH"><img src="https://hackmd.io/badge.svg" alt="Meeting notes" /></a></p>
|
|
<h2 id="is-rtic-an-rtos"><a class="header" href="#is-rtic-an-rtos">Is RTIC an RTOS?</a></h2>
|
|
<p>A common question is whether RTIC is an RTOS or not, and depending on your background the answer may vary. From RTIC's developers point of view; RTIC is a hardware accelerated RTOS that utilizes the hardware such as the NVIC on Cortex-M MCUs, CLIC on RISC-V etc. to perform scheduling, rather than the more classical software kernel.</p>
|
|
<p>Another common view from the community is that RTIC is a concurrency framework as there
|
|
is no software kernel and that it relies on external HALs.</p>
|
|
<h2 id="rtic---the-past-current-and-future"><a class="header" href="#rtic---the-past-current-and-future">RTIC - The Past, current and Future</a></h2>
|
|
<p>This section gives a background to the RTIC model. Feel free to skip to section <a href="preface.html#rtic-the-model">RTIC the model</a> for a TL;DR.</p>
|
|
<p>The RTIC framework takes the outset from real-time systems research at Luleå University of Technology (LTU) Sweden. RTIC is inspired by the concurrency model of the <a href="https://web.archive.org/web/20230325133224/http://timber-lang.org/">Timber</a> language, the <a href="https://www.diva-portal.org/smash/get/diva2:1005680/FULLTEXT01.pdf">RTFM-SRP</a> based scheduler, the <a href="https://ltu.diva-portal.org/smash/get/diva2:1013248/FULLTEXT01.pdf">RTFM-core</a> language and <a href="https://ltu.diva-portal.org/smash/get/diva2:1013030/FULLTEXT01.pdf">Abstract Timer</a> implementation. For a full list of related research see <a href="http://ltu.diva-portal.org/smash/resultList.jsf?query=RTFM&language=en&searchType=SIMPLE&noOfRows=50&sortOrder=author_sort_asc&sortOrder2=title_sort_asc&onlyFullText=false&sf=all&aq=%5B%5B%5D%5D&aqe=%5B%5D&aq2=%5B%5B%5D%5D&af=%5B%5D">RTFM</a> and <a href="http://ltu.diva-portal.org/smash/resultList.jsf?query=RTIC&language=en&searchType=SIMPLE&noOfRows=50&sortOrder=author_sort_asc&sortOrder2=title_sort_asc&onlyFullText=false&sf=all&aq=%5B%5B%5D%5D&aqe=%5B%5D&aq2=%5B%5B%5D%5D&af=%5B%5D">RTIC</a> publications.</p>
|
|
<h2 id="stack-resource-policy-based-scheduling"><a class="header" href="#stack-resource-policy-based-scheduling">Stack Resource Policy based Scheduling</a></h2>
|
|
<p><a href="https://link.springer.com/article/10.1007/BF00365393">Stack Resource Policy (SRP)</a> based concurrency and resource management is at heart of the RTIC framework. The SRP model itself extends on <a href="https://ieeexplore.ieee.org/document/57058">Priority Inheritance Protocols</a>, and provides a set of outstanding properties for single core scheduling. To name a few:</p>
|
|
<ul>
|
|
<li>preemptive deadlock and race-free scheduling</li>
|
|
<li>resource efficiency
|
|
<ul>
|
|
<li>tasks execute on a single shared stack</li>
|
|
<li>tasks run-to-completion with wait free access to shared resources</li>
|
|
</ul>
|
|
</li>
|
|
<li>predictable scheduling, with bounded priority inversion by a single (named) critical section</li>
|
|
<li>theoretical underpinning amenable to static analysis (e.g., for task response times and overall schedulability)</li>
|
|
</ul>
|
|
<p>SRP comes with a set of system-wide requirements:</p>
|
|
<ul>
|
|
<li>each task is associated a static priority,</li>
|
|
<li>tasks execute on a single-core,</li>
|
|
<li>tasks must be run-to-completion, and</li>
|
|
<li>resources must be claimed/locked in LIFO order.</li>
|
|
</ul>
|
|
<h2 id="srp-analysis"><a class="header" href="#srp-analysis">SRP analysis</a></h2>
|
|
<p>SRP based scheduling requires the set of static priority tasks and their access to shared resources to be known in order to compute a static <em>ceiling</em> (𝝅) for each resource. The static resource <em>ceiling</em> 𝝅(r) reflects the maximum static priority of any task that accesses the resource <code>r</code>.</p>
|
|
<h3 id="example"><a class="header" href="#example">Example</a></h3>
|
|
<p>Assume two tasks <code>A</code> (with priority <code>p(A) = 2</code>) and <code>B</code> (with priority <code>p(B) = 4</code>) both accessing the shared resource <code>R</code>. The static ceiling of <code>R</code> is 4 (computed from <code>𝝅(R) = max(p(A) = 2, p(B) = 4) = 4</code>).</p>
|
|
<p>A graph representation of the example:</p>
|
|
<pre class="mermaid">graph LR
|
|
A["p(A) = 2"] --> R
|
|
B["p(B) = 4"] --> R
|
|
R["𝝅(R) = 4"]
|
|
</pre>
|
|
<h2 id="rtic-the-hardware-accelerated-real-time-scheduler"><a class="header" href="#rtic-the-hardware-accelerated-real-time-scheduler">RTIC the hardware accelerated real-time scheduler</a></h2>
|
|
<p>SRP itself is compatible with both dynamic and static priority scheduling. For the implementation of RTIC we leverage on the underlying hardware for accelerated static priority scheduling.</p>
|
|
<p>In the case of the <code>ARM Cortex-M</code> architecture, each interrupt vector entry <code>v[i]</code> is associated a function pointer (<code>v[i].fn</code>), and a static priority (<code>v[i].priority</code>), an enabled- (<code>v[i].enabled</code>) and a pending-bit (<code>v[i].pending</code>).</p>
|
|
<p>An interrupt <code>i</code> is scheduled (run) by the hardware under the conditions:</p>
|
|
<ol>
|
|
<li>is <code>pended</code> and <code>enabled</code> and has a priority higher than the (optional <code>BASEPRI</code>) register, and</li>
|
|
<li>has the highest priority among interrupts meeting 1.</li>
|
|
</ol>
|
|
<p>The first condition (1) can be seen a filter allowing RTIC to take control over which tasks should be allowed to start (and which should be prevented from starting).</p>
|
|
<p>The SPR model for single-core static scheduling on the other hand states that a task should be scheduled (run) under the conditions:</p>
|
|
<ol>
|
|
<li>it is <code>requested</code> to run and has a static priority higher than the current system ceiling (𝜫)</li>
|
|
<li>it has the highest static priority among tasks meeting 1.</li>
|
|
</ol>
|
|
<p>The similarities are striking and it is not by chance/luck/coincidence. The hardware was cleverly designed with real-time scheduling in mind.</p>
|
|
<p>In order to map the SRP scheduling onto the hardware we need to take a closer look at the system ceiling (𝜫). Under SRP 𝜫 is computed as the maximum priority ceiling of the currently held resources, and will thus change dynamically during the system operation.</p>
|
|
<h2 id="example-1"><a class="header" href="#example-1">Example</a></h2>
|
|
<p>Assume the task model above. Starting from an idle system, 𝜫 is 0, (no task is holding any resource). Assume that <code>A</code> is requested for execution, it will immediately be scheduled. Assume that <code>A</code> claims (locks) the resource <code>R</code>. During the claim (lock of <code>R</code>) any request <code>B</code> will be blocked from starting (by 𝜫 = <code>max(𝝅(R) = 4) = 4</code>, <code>p(B) = 4</code>, thus SRP scheduling condition 1 is not met).</p>
|
|
<h2 id="mapping"><a class="header" href="#mapping">Mapping</a></h2>
|
|
<p>The mapping of static priority SRP based scheduling to the Cortex M hardware is straightforward:</p>
|
|
<ul>
|
|
<li>each task <code>t</code> are mapped to an interrupt vector index <code>i</code> with a corresponding function <code>v[i].fn = t</code> and given the static priority <code>v[i].priority = p(t)</code>.</li>
|
|
<li>the current system ceiling is mapped to the <code>BASEPRI</code> register or implemented through masking the interrupt enable bits accordingly.</li>
|
|
</ul>
|
|
<h2 id="example-2"><a class="header" href="#example-2">Example</a></h2>
|
|
<p>For the running example, a snapshot of the ARM Cortex M <a href="https://developer.arm.com/documentation/ddi0337/h/nested-vectored-interrupt-controller/about-the-nvic">Nested Vectored Interrupt Controller (NVIC)</a> may have the following configuration (after task <code>A</code> has been pended for execution.)</p>
|
|
<div class="table-wrapper"><table><thead><tr><th>Index</th><th>Fn</th><th>Priority</th><th>Enabled</th><th>Pended</th></tr></thead><tbody>
|
|
<tr><td>0</td><td>A</td><td>2</td><td>true</td><td>true</td></tr>
|
|
<tr><td>1</td><td>B</td><td>4</td><td>true</td><td>false</td></tr>
|
|
</tbody></table>
|
|
</div>
|
|
<p>(As discussed later, the assignment of interrupt and exception vectors is up to the user.)</p>
|
|
<p>A claim (lock(r)) will change the current system ceiling (𝜫) and can be implemented as a <em>named</em> critical section:</p>
|
|
<ul>
|
|
<li>old_ceiling = 𝜫, 𝜫 = 𝝅(r)</li>
|
|
<li>execute code within critical section</li>
|
|
<li>old_ceiling = 𝜫</li>
|
|
</ul>
|
|
<p>This amounts to a resource protection mechanism requiring only two machine instructions on enter and one on exit the critical section for managing the <code>BASEPRI</code> register. For architectures lacking <code>BASEPRI</code>, we can implement the system ceiling through a set of machine instructions for disabling/enabling interrupts on entry/exit for the named critical section. The number of machine instructions vary depending on the number of mask registers that needs to be updated (a single machine operation can operate on up to 32 interrupts, so for the M0/M0+ architecture a single instruction suffice). RTIC will determine the ceiling values and masking constants at compile time, thus all operations is in Rust terms zero-cost.</p>
|
|
<p>In this way RTIC fuses SRP based preemptive scheduling with a zero-cost hardware accelerated implementation, resulting in "best in class" guarantees and performance.</p>
|
|
<p>Given that the approach is dead simple, how come SRP and hardware accelerated scheduling is not adopted by any other mainstream RTOS?</p>
|
|
<p>The answer is simple, the commonly adopted threading model does not lend itself well to static analysis - there is no known way to extract the task/resource dependencies from the source code at compile time (thus ceilings cannot be efficiently computed and the LIFO resource locking requirement cannot be ensured). Thus, SRP based scheduling is in the general case out of reach for any thread based RTOS.</p>
|
|
<h2 id="rtic-into-the-future"><a class="header" href="#rtic-into-the-future">RTIC into the Future</a></h2>
|
|
<p>Asynchronous programming in various forms are getting increased popularity and language support. Rust natively provides an <code>async</code>/<code>await</code> API for cooperative multitasking and the compiler generates the necessary boilerplate for storing and retrieving execution contexts (i.e., managing the set of local variables that spans each <code>await</code>).</p>
|
|
<p>The Rust standard library provides collections for dynamically allocated data-structures which are useful to manage execution contexts at run-time. However, in the setting of resource constrained real-time systems, dynamic allocations are problematic (both regarding performance and reliability - Rust runs into a <em>panic</em> on an out-of-memory condition). Thus, static allocation is the preferable approach!</p>
|
|
<p>From a modelling perspective <code>async/await</code> lifts the run-to-completion requirement of SRP, and each section of code between two yield points (<code>await</code>s) can be seen as an individual task. The compiler will reject any attempt to <code>await</code> while holding a resource (not doing so would break the strict LIFO requirement on resource usage under SRP).</p>
|
|
<p>So with the technical stuff out of the way, what does <code>async/await</code> bring to the table?</p>
|
|
<p>The answer is - improved ergonomics! A recurring use case is to have task perform a sequence of requests and then await their results in order to progress. Without <code>async</code>/<code>await</code> the programmer would be forced to split the task into individual sub-tasks and maintain some sort of state encoding (and manually progress by selecting sub-task). Using <code>async/await</code> each yield point (<code>await</code>) essentially represents a state, and the progression mechanism is built automatically for you at compile time by means of <code>Futures</code>.</p>
|
|
<p>Rust <code>async</code>/<code>await</code> support is still incomplete and/or under development (e.g., there are no stable way to express <code>async</code> closures, precluding use in iterator patterns). Nevertheless, Rust <code>async</code>/<code>await</code> is production ready and covers most common use cases.</p>
|
|
<p>An important property is that futures are composable, thus you can await either, all, or any combination of possible futures (allowing e.g., timeouts and/or asynchronous errors to be promptly handled).</p>
|
|
<h2 id="rtic-the-model"><a class="header" href="#rtic-the-model">RTIC the model</a></h2>
|
|
<p>An RTIC <code>app</code> is a declarative and executable system model for single-core applications, defining a set of (<code>local</code> and <code>shared</code>) resources operated on by a set of (<code>init</code>, <code>idle</code>, <em>hardware</em> and <em>software</em>) tasks. In short the <code>init</code> task runs before any other task returning a set of resources (<code>local</code> and <code>shared</code>). Tasks run preemptively based on their associated static priority, <code>idle</code> has the lowest priority (and can be used for background work, and/or to put the system to sleep until woken by some event). Hardware tasks are bound to underlying hardware interrupts, while software tasks are scheduled by asynchronous executors (one for each software task priority).</p>
|
|
<p>At compile time the task/resource model is analyzed under SRP and executable code generated with the following outstanding properties:</p>
|
|
<ul>
|
|
<li>guaranteed race-free resource access and deadlock-free execution on a single-shared stack (thanks to SRP)
|
|
<ul>
|
|
<li>hardware task scheduling is performed directly by the hardware, and</li>
|
|
<li>software task scheduling is performed by auto generated async executors tailored to the application.</li>
|
|
</ul>
|
|
</li>
|
|
</ul>
|
|
<p>The RTIC API design ensures that both SRP requirements and Rust soundness rules are upheld at all times, thus the executable model is correct by construction. Overall, the generated code infers no additional overhead in comparison to a handwritten implementation, thus in Rust terms RTIC offers a zero-cost abstraction to concurrency.</p>
|
|
|
|
</main>
|
|
|
|
<nav class="nav-wrapper" aria-label="Page navigation">
|
|
<!-- Mobile navigation buttons -->
|
|
|
|
<a rel="next prefetch" href="starting_a_project.html" class="mobile-nav-chapters next" title="Next chapter" aria-label="Next chapter" aria-keyshortcuts="Right">
|
|
<i class="fa fa-angle-right"></i>
|
|
</a>
|
|
|
|
<div style="clear: both"></div>
|
|
</nav>
|
|
</div>
|
|
</div>
|
|
|
|
<nav class="nav-wide-wrapper" aria-label="Page navigation">
|
|
|
|
<a rel="next prefetch" href="starting_a_project.html" class="nav-chapters next" title="Next chapter" aria-label="Next chapter" aria-keyshortcuts="Right">
|
|
<i class="fa fa-angle-right"></i>
|
|
</a>
|
|
</nav>
|
|
|
|
</div>
|
|
|
|
|
|
|
|
|
|
<script>
|
|
window.playground_copyable = true;
|
|
</script>
|
|
|
|
|
|
<script src="elasticlunr.min.js"></script>
|
|
<script src="mark.min.js"></script>
|
|
<script src="searcher.js"></script>
|
|
|
|
<script src="clipboard.min.js"></script>
|
|
<script src="highlight.js"></script>
|
|
<script src="book.js"></script>
|
|
|
|
<!-- Custom JS scripts -->
|
|
<script src="mermaid.min.js"></script>
|
|
<script src="mermaid-init.js"></script>
|
|
|
|
|
|
</div>
|
|
</body>
|
|
</html>
|