Compare commits
No commits in common. "master" and "dockerfile-fixes" have entirely different histories.
master
...
dockerfile
|
|
@ -1,12 +1,12 @@
|
||||||
[submodule "vm-sdk"]
|
[submodule "vm-sdk"]
|
||||||
path = vm-sdk
|
path = vm-sdk
|
||||||
url = https://github.com/parazyd/vm-sdk.git
|
url = https://git.devuan.org/sdk/vm-sdk.git
|
||||||
[submodule "arm-sdk"]
|
[submodule "arm-sdk"]
|
||||||
path = arm-sdk
|
path = arm-sdk
|
||||||
url = https://github.com/parazyd/arm-sdk.git
|
url = https://git.devuan.org/sdk/arm-sdk.git
|
||||||
[submodule "live-sdk"]
|
[submodule "live-sdk"]
|
||||||
path = live-sdk
|
path = live-sdk
|
||||||
url = https://github.com/parazyd/live-sdk.git
|
url = https://git.devuan.org/sdk/live-sdk.git
|
||||||
[submodule "docs/webnomad"]
|
[submodule "docs/webnomad"]
|
||||||
path = docs/webnomad
|
path = docs/webnomad
|
||||||
url = https://github.com/dyne/webnomad
|
url = https://github.com/dyne/webnomad
|
||||||
|
|
|
||||||
12
README.md
12
README.md
|
|
@ -11,7 +11,7 @@ peer-to-peer network cluster.
|
||||||
|
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<img src="https://www.dyne.org/wp-content/uploads/2019/12/decode-os_only-logo.png" class="pic" alt="DECODE OS logo">
|
<img src="https://decodeos.dyne.org/img/decodeos_logo-800px.jpg" class="pic" alt="DECODE OS logo">
|
||||||
|
|
||||||
| Features | Components |
|
| Features | Components |
|
||||||
|--------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
|--------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
||||||
|
|
@ -22,7 +22,6 @@ peer-to-peer network cluster.
|
||||||
| Pluggable consensus algorithm | [Redis](https://redis.io) based consensus broker |
|
| Pluggable consensus algorithm | [Redis](https://redis.io) based consensus broker |
|
||||||
| Read-only and authenticated system | [SquashFS](http://tldp.org/HOWTO/SquashFS-HOWTO/whatis.html) + [overlayfs](https://www.kernel.org/doc/Documentation/filesystems/overlayfs.txt) + [Btrfs](https://btrfs.wiki.kernel.org/index.php/Main_Page) |
|
| Read-only and authenticated system | [SquashFS](http://tldp.org/HOWTO/SquashFS-HOWTO/whatis.html) + [overlayfs](https://www.kernel.org/doc/Documentation/filesystems/overlayfs.txt) + [Btrfs](https://btrfs.wiki.kernel.org/index.php/Main_Page) |
|
||||||
| Integrated updating mechanism | [Roundshot](https://github.com/DECODEproject/roundshot) initramfs |
|
| Integrated updating mechanism | [Roundshot](https://github.com/DECODEproject/roundshot) initramfs |
|
||||||
| Built-in Graphical dashboard | [Netdata](https://github.com/netdata/netdata) resource monitor |
|
|
||||||
| Low power consumption, outdoor usage | Ports to embedded ARM boards |
|
| Low power consumption, outdoor usage | Ports to embedded ARM boards |
|
||||||
| Extensible platform support | Includes latest JDK, Golang, Python etc. |
|
| Extensible platform support | Includes latest JDK, Golang, Python etc. |
|
||||||
| Minimal resource consumption | Online with less than 64MB of RAM |
|
| Minimal resource consumption | Online with less than 64MB of RAM |
|
||||||
|
|
@ -114,20 +113,13 @@ git pull origin master && git submodule update --init --recursive --checkout
|
||||||
|
|
||||||
### Building for ARM targets
|
### Building for ARM targets
|
||||||
|
|
||||||
A more detailed reference for the arm-sdk can be found here: https://git.devuan.org/sdk/arm-sdk
|
|
||||||
|
|
||||||
```
|
```
|
||||||
cd arm-sdk # (or vm-sdk or live-sdk depending from your target)
|
cd arm-sdk # (or vm-sdk or live-sdk depending from your target)
|
||||||
zsh -f
|
|
||||||
./init.sh # and when this is done, execute the command in the bottom of the output
|
|
||||||
source sdk
|
source sdk
|
||||||
load devuan raspi3 decode # (replace "raspi3" with your board name, from the list below)
|
load devuan sunxi decode # (specific to the arm-sdk)
|
||||||
bootstrap_complete_base
|
bootstrap_complete_base
|
||||||
```
|
```
|
||||||
|
|
||||||
Here is the list of the supported boxes: https://git.devuan.org/sdk/arm-sdk/blob/master/sdk
|
|
||||||
|
|
||||||
|
|
||||||
### Building for VM targets
|
### Building for VM targets
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
2
arm-sdk
2
arm-sdk
|
|
@ -1 +1 @@
|
||||||
Subproject commit 5ea550fe6ce8adcba8eac84c212cb8c49d8626f6
|
Subproject commit 926680290f3cdaf8bc1ce895d064b68910d848b3
|
||||||
|
|
@ -317,14 +317,14 @@ deb http://pkgmaster.devuan.org/merged ascii main
|
||||||
deb http://pkgmaster.devuan.org/merged ascii-updates main
|
deb http://pkgmaster.devuan.org/merged ascii-updates main
|
||||||
deb http://pkgmaster.devuan.org/merged ascii-security main
|
deb http://pkgmaster.devuan.org/merged ascii-security main
|
||||||
deb http://deb.torproject.org/torproject.org stretch main
|
deb http://deb.torproject.org/torproject.org stretch main
|
||||||
deb http://deb.torproject.org/torproject.org tor-experimental-0.4.0.x-stretch main
|
deb http://deb.torproject.org/torproject.org tor-experimental-0.3.4.x-stretch main
|
||||||
|
|
||||||
## source repositories
|
## source repositories
|
||||||
#deb-src http://pkgmaster.devuan.org/merged ascii main
|
#deb-src http://pkgmaster.devuan.org/merged ascii main
|
||||||
#deb-src http://pkgmaster.devuan.org/merged ascii-updates main
|
#deb-src http://pkgmaster.devuan.org/merged ascii-updates main
|
||||||
#deb-src http://pkgmaster.devuan.org/merged ascii-security main
|
#deb-src http://pkgmaster.devuan.org/merged ascii-security main
|
||||||
#deb-src http://deb.torproject.org/torproject.org stretch main
|
#deb-src http://deb.torproject.org/torproject.org stretch main
|
||||||
#deb-src http://deb.torproject.org/torproject.org tor-experimental-0.4.0.x-stretch main
|
#deb-src http://deb.torproject.org/torproject.org tor-experimental-0.3.4.x-stretch main
|
||||||
EOF
|
EOF
|
||||||
}
|
}
|
||||||
## }}}
|
## }}}
|
||||||
|
|
|
||||||
|
|
@ -1,55 +0,0 @@
|
||||||
Stable DECODE OS release
|
|
||||||
========================
|
|
||||||
|
|
||||||
This document accompanies the stable release of the DECODE OS, one of
|
|
||||||
the core development outputs of the DECODE project, aimed at providing
|
|
||||||
a reliable operating system to run application space development in an
|
|
||||||
environment ensuring privacy by design outside of the application
|
|
||||||
domain. This deliverable references, without duplication of
|
|
||||||
information, the research and development done and detailed in
|
|
||||||
previous deliverables D4.1 and D4.4.
|
|
||||||
|
|
||||||
The DECODE OS is a GNU+Linux distribution based on Devuan.org to
|
|
||||||
provide a minimalist base for distributed computing micro-services
|
|
||||||
capable of targeting any mainstream hardware platform, from
|
|
||||||
virtual-machines to ARM boards to bare-metal server racks.
|
|
||||||
|
|
||||||
The main website for this distribution is https://decodeos.dyne.org
|
|
||||||
|
|
||||||
|
|
||||||
As part of the DECODE OS distribution, backend software applications
|
|
||||||
have been developed to implement
|
|
||||||
|
|
||||||
1. a front-end web application to facilitate the adoption of the
|
|
||||||
DECODE continuous integration infrastructure (toaster)
|
|
||||||
https://toaster.dyne.org
|
|
||||||
2. a continuous integration system to release and customize new
|
|
||||||
versions of DECODE OS (SDK) https://git.devuan.org/sdk
|
|
||||||
3. a private peer-to-peer network over the Tor protocol (tor-dam)
|
|
||||||
https://github.com/decodeproject/tor-dam
|
|
||||||
|
|
||||||
These core features of these three components will be described in the
|
|
||||||
following sections of this document, along with operational
|
|
||||||
instructions.
|
|
||||||
|
|
||||||
Due to the experimental stage of development of other components in
|
|
||||||
DECODE and according to the LEAN principles declared in the project,
|
|
||||||
this stable release doesn't only constitute a final point of arrival
|
|
||||||
for this development task. What DECODE OS can do today is facilitating
|
|
||||||
the deployment of lab-tested software applications (for example made
|
|
||||||
in a Docker format, widely adopted by other partners in DECODE) and
|
|
||||||
render these prototypes into a production ready format that can be
|
|
||||||
deployed on the open-hardware DECODE BOX as well on virtual-machines.
|
|
||||||
|
|
||||||
We consider this achievement highly beneficial for a project whose
|
|
||||||
development is still in-flux, as well for the free and open source
|
|
||||||
community out there, since the access to the powerful features of the
|
|
||||||
SDK is now made very easy via an integrated continuous pipeline.
|
|
||||||
|
|
||||||
In light of these advantages, there is a clear intention within our
|
|
||||||
organisation (mainly by DYNE) to keep maintaining DECODE OS also
|
|
||||||
beyond the span of the project and this very task now concluded, since
|
|
||||||
it greatly helps the manning of prototypes into stable production
|
|
||||||
environments.
|
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -1,293 +0,0 @@
|
||||||
The DECODE SDK
|
|
||||||
==============
|
|
||||||
|
|
||||||
The DECODE SDK is a unique build framework written to ease maintenance
|
|
||||||
and production of various types of the Devuan distribution images,
|
|
||||||
such as: live ISOs, virtual machine images, and images targeted at
|
|
||||||
embedded ARM boards. This section explains how to use the SDK, gives
|
|
||||||
and inside look at its various parts and documents the workflow to be
|
|
||||||
used when modifying its code.
|
|
||||||
|
|
||||||
The SDK is designed in such a way that there are levels of priority
|
|
||||||
within the scripts. First there is `libdevuansdk`, which holds the
|
|
||||||
vanilla configuration, then come the various wrappers targeted around
|
|
||||||
specific targets (`live`, `virtual`, `embedded`), and afterwards we
|
|
||||||
optionally add more on top of it if we need to customize or override
|
|
||||||
specific functions. This is for example the case with DECODE OS,
|
|
||||||
where we have to add additional software and extra components on top
|
|
||||||
of the base Devuan system.
|
|
||||||
|
|
||||||
|
|
||||||
libdevuansdk
|
|
||||||
------------
|
|
||||||
|
|
||||||
_libdevuansdk_ is the core of any part of the Devuan SDK. It holds the
|
|
||||||
common knowledge between all of the upper wrappers such as _live-sdk_,
|
|
||||||
_vm-sdk_, and _arm-sdk_. Simply put, it is a shell script library to
|
|
||||||
unify the use and creation of various functions spread throughout the
|
|
||||||
complete Devuan SDK.
|
|
||||||
|
|
||||||
The wrappers are designed to be used interactively from a terminal, as
|
|
||||||
well as automated from shell scripts. _libdevuansdk_ uses an
|
|
||||||
additional _zsh_ library called [zuper](https://github.com/dyne/zuper)
|
|
||||||
to ease the variable declaration and scoping, as well as error
|
|
||||||
checking and debugging. However, _zuper_ is not included in
|
|
||||||
_libdevuansdk_ itself - one is required to include it in its
|
|
||||||
respective wrapper. _live-sdk_, _vm-sdk_, and _arm-sdk_ can be taken
|
|
||||||
as example. libdevuansdk itself has some software dependencies that
|
|
||||||
should be installed prior to use:
|
|
||||||
|
|
||||||
```
|
|
||||||
zsh
|
|
||||||
debootstrap
|
|
||||||
sudo
|
|
||||||
kpartx
|
|
||||||
cgpt
|
|
||||||
xz-utils
|
|
||||||
```
|
|
||||||
|
|
||||||
|
|
||||||
### Workflow
|
|
||||||
|
|
||||||
Working with _libdevuansdk_ splits into categories of what you want to
|
|
||||||
do. _zlibs_ are files separated into the following categories:
|
|
||||||
|
|
||||||
* ***bootstrap*** Contains the functions for the bootstrap process.
|
|
||||||
Creating a minimal debootstrap base system, and making it into a
|
|
||||||
compressed file (tar.gz) for later use so one does not have to wait
|
|
||||||
for the lengthy bootstrap process on each consequent build.
|
|
||||||
|
|
||||||
* ***helpers*** Contains the helper functions for _libdevuansdk_ that
|
|
||||||
make the workflow a bit easier to use and handle.
|
|
||||||
|
|
||||||
* ***imaging*** Contains the functions necessary for creating raw
|
|
||||||
dd-able images.
|
|
||||||
|
|
||||||
* ***rsync*** Contains rsync and file copying functions.
|
|
||||||
|
|
||||||
* ***sysconf*** Contains the default system configuration.
|
|
||||||
|
|
||||||
|
|
||||||
### Usage
|
|
||||||
|
|
||||||
As libdevuansdk is not very useful when invoked on its own, its usage
|
|
||||||
will be explained at later parts, for each specific wrapper. The
|
|
||||||
technical documentation of _libdevuansdk_ will follow in its
|
|
||||||
appropriate section.
|
|
||||||
|
|
||||||
|
|
||||||
The wrappers
|
|
||||||
------------
|
|
||||||
|
|
||||||
As mentioned, _libdevuansdk_ is the core library we wrap around. The
|
|
||||||
currently existing wrappers are called _live-sdk_, _vm-sdk_, and
|
|
||||||
_arm-sdk_. These facilitate the builds of liveCDs, virtual machines, and
|
|
||||||
images for embedded ARM devices, respectively. Each of them have their
|
|
||||||
own section in this paper.
|
|
||||||
|
|
||||||
Since all of these wrappers, along with _libdevuansdk_, hold a
|
|
||||||
_vanilla_ Devuan configuration, it is best to keep their code
|
|
||||||
untouched. To allow for custom configurations, we introduced a concept
|
|
||||||
called *blends*. Blends are a simple way to customize the base image
|
|
||||||
of the OS-to-be before building it, allowing to easily add packages,
|
|
||||||
kernels, and virtually anything one might want to do in the
|
|
||||||
image. This exactly is the case with DECODE OS.
|
|
||||||
|
|
||||||
|
|
||||||
arm-sdk
|
|
||||||
-------
|
|
||||||
|
|
||||||
The _arm-sdk_ is our way of facilitating builds for embedded ARM boards
|
|
||||||
such as Allwinner-based CPUs, Raspberry Pis, Chromebooks, etc. It holds
|
|
||||||
a knowledgebase for a number of embedded devices, and how to build
|
|
||||||
according kernels and bootloaders.
|
|
||||||
|
|
||||||
|
|
||||||
### Directory structure
|
|
||||||
|
|
||||||
_arm-sdk_'s directory structure is separated into places where we hold
|
|
||||||
our boards and their kernel configurations, device-specific
|
|
||||||
directories with firmware and/or configuration, and a _lib_ directory
|
|
||||||
(where we keep _libdevuansdk_ and the like).
|
|
||||||
|
|
||||||
|
|
||||||
### Obtaining arm-sdk
|
|
||||||
|
|
||||||
The SDK, like any other part of Devuan's software toolchain, should be
|
|
||||||
obtained via _git_. The repositories are hosted on Devuan's Gitlab. To
|
|
||||||
grab it, we simply issue a _git clone_ command on a terminal, and
|
|
||||||
since it contains linked git submodules - we append _--recursive_ to
|
|
||||||
it:
|
|
||||||
|
|
||||||
```
|
|
||||||
$ git clone https://git.devuan.org/sdk/arm-sdk --recursive
|
|
||||||
```
|
|
||||||
|
|
||||||
Consult the _README.md_ file found in this repository to see what are
|
|
||||||
the required dependencies to use _arm-sdk_.
|
|
||||||
|
|
||||||
|
|
||||||
### Using arm-sdk
|
|
||||||
|
|
||||||
Once the build system is obtained, it can now be used interactively. The
|
|
||||||
process is very simple, and to build an image one can actually use a
|
|
||||||
single shell command. However, we shall first show how it works.
|
|
||||||
|
|
||||||
In _arm-sdk_, every board has its own script located in the _boards_
|
|
||||||
directory. In most cases, these scripts contain functions to build the
|
|
||||||
Linux kernel, and a bootloader needed for the board to boot. This is
|
|
||||||
the only difference between all the boards, which requires every board
|
|
||||||
to have their own script. We are able to reuse the _rootfs_ that was
|
|
||||||
bootstrapped before. For our example, let's take the _Nokia N900_
|
|
||||||
build script. To build a _vanilla_ image for it, we simply issue:
|
|
||||||
|
|
||||||
|
|
||||||
```
|
|
||||||
$ zsh -f -c 'source sdk && load devuan n900 && build_image_dist'
|
|
||||||
|
|
||||||
```
|
|
||||||
|
|
||||||
This will fire up the build process, and after a certain amount of time
|
|
||||||
we will have our compressed image ready and checksummed inside the
|
|
||||||
_dist_ directory.
|
|
||||||
|
|
||||||
The oneliner above is self-explanatory: We first start a new untainted
|
|
||||||
shell, source the sdk file to get an interactive SDK shell, then we
|
|
||||||
initialize the operating system along with the board we are building,
|
|
||||||
and finally we issue a helper command that calls all the necessary
|
|
||||||
functions to build our image. The _load_ command takes an optional
|
|
||||||
third argument which is the name of our _blend_ (the way to customize
|
|
||||||
our _vanilla_ image) which will be explained later. So in this case,
|
|
||||||
our oneliner would look like:
|
|
||||||
|
|
||||||
```
|
|
||||||
$ zsh -f -c 'source sdk && load devuan n900 decode && build_image_dist'
|
|
||||||
```
|
|
||||||
|
|
||||||
This would create an image with the _"decode"_ blend, which is available
|
|
||||||
by cloning the DECODE OS git repository. The *build_image_dist* command
|
|
||||||
is a helper function located in _libdevuansdk_ that wraps around the 8
|
|
||||||
functions needed to build our image. They are all explained in the
|
|
||||||
technical part of this paper.
|
|
||||||
|
|
||||||
|
|
||||||
live-sdk
|
|
||||||
--------
|
|
||||||
|
|
||||||
The _live-sdk_ is used to build bootable images, better known as Live
|
|
||||||
CDs. Its structure is very similar to _vm-sdk_ and is a lot smaller than
|
|
||||||
_arm-sdk_.
|
|
||||||
|
|
||||||
|
|
||||||
### Directory structure
|
|
||||||
|
|
||||||
Unlike _arm-sdk_, in _live-sdk_ we have no need for specific boards or
|
|
||||||
setups, so in this case we only host the interactive shell init, and
|
|
||||||
libraries.
|
|
||||||
|
|
||||||
|
|
||||||
### Obtaining live-sdk
|
|
||||||
|
|
||||||
The SDK, like any other, should be obtained via _git_. The repositories
|
|
||||||
are hosted on Devuan's Gitlab. To grab it, we simply issue a _git clone_
|
|
||||||
command, an since it contains git submodules - we append _--recursive_
|
|
||||||
to it:
|
|
||||||
|
|
||||||
```
|
|
||||||
$ git clone https://git.devuan.org/sdk/live-sdk --recursive
|
|
||||||
```
|
|
||||||
|
|
||||||
Consult the _README.md_ file found in this repository to see what are
|
|
||||||
the required dependencies to use _live-sdk_.
|
|
||||||
|
|
||||||
|
|
||||||
### Using live-sdk
|
|
||||||
|
|
||||||
Much like _arm-sdk_, the _live-sdk_ is used the same way. With two
|
|
||||||
specific differences. Since we don't have any need for specific
|
|
||||||
boards, when loading we don't specify a board, but rather the CPU
|
|
||||||
architecture we are building for. Currently supported are *i386* and
|
|
||||||
*amd64* which represent 32bit and 64bit respectively. To build a
|
|
||||||
_vanilla_ live ISO, we issue:
|
|
||||||
|
|
||||||
```
|
|
||||||
$ zsh -f -c 'source sdk && load devuan amd64 && build_iso_dist'
|
|
||||||
```
|
|
||||||
|
|
||||||
This will start the build process, and after a certain amount of time we
|
|
||||||
will have our ISO ready and inside the _dist_ directory.
|
|
||||||
|
|
||||||
Just like in _arm-sdk_, we can use a _blend_ and customize our OS:
|
|
||||||
|
|
||||||
```
|
|
||||||
$ zsh -f -c 'source sdk && load devuan amd64 decode && build_iso_dist'
|
|
||||||
```
|
|
||||||
|
|
||||||
So this would create a live ISO of DECODE OS. Again as noted, this can
|
|
||||||
be obtained by recursively cloning the corresponding (DECODE-OS) git
|
|
||||||
repository.
|
|
||||||
|
|
||||||
The *build_iso_dist* command is a helper function located in
|
|
||||||
_libdevuansdk_ that wraps around the 9 functions needed to build our
|
|
||||||
image. They are all explained in the technical part of this manual.
|
|
||||||
|
|
||||||
|
|
||||||
vm-sdk
|
|
||||||
------
|
|
||||||
|
|
||||||
The _vm-sdk_ is used to build VirtualBox/Vagrant boxes, and virtual
|
|
||||||
images for emulation, in QCOW2 format, which is the byproduct of
|
|
||||||
building a Vagrant box. Its structure is very similar to _live-sdk_
|
|
||||||
and is the smallest of the three wrappers currently found in the
|
|
||||||
Devuan SDK.
|
|
||||||
|
|
||||||
|
|
||||||
### Directory structure
|
|
||||||
|
|
||||||
Like with _live-sdk_, in _vm-sdk_ we have no need for specific boards
|
|
||||||
or setups, so in this case we only host the interactive shell init,
|
|
||||||
and libraries.
|
|
||||||
|
|
||||||
|
|
||||||
### Obtaining vm-sdk
|
|
||||||
|
|
||||||
The SDK, like any other, should be obtained via _git_. The
|
|
||||||
repositories are hosted on Devuan's Gitlab. To grab it, we simply
|
|
||||||
issue a _git clone_ command, an since it contains git submodules - we
|
|
||||||
append _--recursive_ to it:
|
|
||||||
|
|
||||||
```
|
|
||||||
$ git clone https://git.devuan.org/sdk/vm-sdk --recursive
|
|
||||||
```
|
|
||||||
|
|
||||||
Consult the _README.md_ file found in this repository to see what are
|
|
||||||
the required dependencies to use _vm-sdk_.
|
|
||||||
|
|
||||||
|
|
||||||
### Using vm-sdk
|
|
||||||
|
|
||||||
Once obtained, we can use it interactively. The process is very simple,
|
|
||||||
and to build an image we use the oneliner we've already seen above.
|
|
||||||
|
|
||||||
Also like with _live-sdk_, we don't build for specific boards, however
|
|
||||||
we also do not create any non-amd64 images, so we don't have to pass
|
|
||||||
an architecture to the load command either. To build a _vanilla_
|
|
||||||
Vagrant Box, VirtualBox image, QCOW2 image, and a cloud-based QCOW2
|
|
||||||
image, we issue:
|
|
||||||
|
|
||||||
```
|
|
||||||
$ zsh -f -c 'source sdk && load devuan && build_vagrant_dist'
|
|
||||||
```
|
|
||||||
|
|
||||||
This line would create all the four types of the VM image.
|
|
||||||
|
|
||||||
As shown with the previous two wrappers, the _blend_ concept works as
|
|
||||||
advertised here as well:
|
|
||||||
|
|
||||||
```
|
|
||||||
$ zsh -f -c 'source sdk && load deuvan decode && build_vagrant_dist'
|
|
||||||
```
|
|
||||||
|
|
||||||
The *build_vagrant_dist* command is a helper function located in
|
|
||||||
_libdevuansdk_ that wraps around the 11 functions needed to build our
|
|
||||||
image. They are all explained in the technical part of this manual.
|
|
||||||
|
|
@ -1,322 +0,0 @@
|
||||||
Blends
|
|
||||||
======
|
|
||||||
|
|
||||||
|
|
||||||
Introduction
|
|
||||||
------------
|
|
||||||
|
|
||||||
In the Devuan SDK, a _blend_ is the preferred way we use to make
|
|
||||||
customizations to the _vanilla_ image. Using blends we can very easily
|
|
||||||
create different flavors of our image, by easily including/excluding
|
|
||||||
certain software packages, files, or anything we wish to do. Blends
|
|
||||||
can become a very quick way of creating entire new derivatives of the
|
|
||||||
original _vanilla_ distribution we are building.
|
|
||||||
|
|
||||||
This time, we will take the DECODE OS as a _blend_ example. In DECODE
|
|
||||||
OS we provide a blend called _decode_ which is the blend we use to
|
|
||||||
create a production release of DECODE OS. The blend's files are
|
|
||||||
contained within their own directory in the _decode-os_ git
|
|
||||||
repository.
|
|
||||||
|
|
||||||
|
|
||||||
Configuration
|
|
||||||
-------------
|
|
||||||
|
|
||||||
Any SDK requires a single file to act as a _blend_. This file is also a
|
|
||||||
_zsh_ script, and, at the very least, it must contain two functions
|
|
||||||
called:
|
|
||||||
|
|
||||||
```
|
|
||||||
blend_preinst()
|
|
||||||
blend_postinst()
|
|
||||||
```
|
|
||||||
|
|
||||||
These functions are your pathway to expanding your blend into whatever
|
|
||||||
you would like it to do. The _preinst_ function is usually called
|
|
||||||
right after bootstrapping the _vanilla_ root filesystem, and the
|
|
||||||
_postinst_ function is called near the very end, just before packing
|
|
||||||
or compressing the image. These two strategic places should be enough
|
|
||||||
to do changes within the image. If this is not enough, blends also
|
|
||||||
allow you to simply **override any variable or function** contained
|
|
||||||
within _libdevuansdk_ or the sdk you are using.
|
|
||||||
|
|
||||||
Our _decode_ blend is such an example. It is a somewhat expanded blend,
|
|
||||||
not contained within a single file, but rather a directory. This allows
|
|
||||||
easier maintenance and makes the scripts clearer and cleaner.
|
|
||||||
|
|
||||||
|
|
||||||
### Adding and removing packages
|
|
||||||
|
|
||||||
When we want to add or remove specific packages to our build, we have
|
|
||||||
to override or append to _libdevuansdk_'s arrays. The array for
|
|
||||||
packages we want installed is called *extra_packages*, and the array
|
|
||||||
for packages we want purged is called *purge_packages*. In the Decode
|
|
||||||
blend, these can be found in the _config_ file located inside the
|
|
||||||
_decode-os_ blend directory. Keep in mind that these arrays could
|
|
||||||
already contain specific packages, so you are advised to rather append
|
|
||||||
to them, than overriding them.
|
|
||||||
|
|
||||||
If the packages you want to install are not available in the
|
|
||||||
repositories, you still have a way of automatically installing
|
|
||||||
them. All you have to do is copy your corresponding .deb files to the
|
|
||||||
following directory of the blend:
|
|
||||||
|
|
||||||
```
|
|
||||||
$R/extra/custom-packages/
|
|
||||||
```
|
|
||||||
|
|
||||||
And when that is done, just call the function *install-custdebs*
|
|
||||||
|
|
||||||
|
|
||||||
Creating a blend
|
|
||||||
----------------
|
|
||||||
|
|
||||||
Rather than explaining the following in theory, you are best off
|
|
||||||
viewing the blend files that are provided with _decode-os_. It is a
|
|
||||||
fairly simple blend and should give you enough insight on how to
|
|
||||||
create your own blend. Here are some important guidelines for creating
|
|
||||||
a blend:
|
|
||||||
|
|
||||||
|
|
||||||
* The blend should always contain at least two functions
|
|
||||||
|
|
||||||
This means you must provide *blend_preinst* and *blend_postinst* in your
|
|
||||||
blend. They don't even have to do anything, but they should be there.
|
|
||||||
These two functions open the path for you to call any other functions
|
|
||||||
you created for your blend.
|
|
||||||
|
|
||||||
|
|
||||||
* When overriding functions, make sure they provide a result that
|
|
||||||
doesn't break the API
|
|
||||||
|
|
||||||
Breaking the API may result in unwanted behavior. You should always
|
|
||||||
study well the functions you are planning to override and figure out if
|
|
||||||
it is safe to override them in the way you want. The same goes for any
|
|
||||||
variables as well.
|
|
||||||
|
|
||||||
|
|
||||||
* Any arguments used after the blend name when loading from the SDK are
|
|
||||||
free for you to use in the blend.
|
|
||||||
|
|
||||||
This means you can use anything after the fourth argument (**$4** in
|
|
||||||
_zsh_) inside your blend if you require passing arguments to it.
|
|
||||||
|
|
||||||
These are some of the more important guidelines. There is plenty more
|
|
||||||
tricks and quirks, but it's easy to find out how to tweak the
|
|
||||||
configuration files and the blend in general once you read through a
|
|
||||||
blend or two on your own.
|
|
||||||
|
|
||||||
|
|
||||||
### Enable the blend
|
|
||||||
|
|
||||||
To use your blend in the first place, you need to make the SDK know
|
|
||||||
about it. Thus you should append the path to your new blend inside
|
|
||||||
the **blend_map** of the _sdk_ file:
|
|
||||||
|
|
||||||
```
|
|
||||||
blend_map=(
|
|
||||||
"devuan-live" "$R/blends/devuan-live/devuan-live.blend"
|
|
||||||
"decode" "$R/../decode.blend"
|
|
||||||
"heads" "$R/../heads.blend"
|
|
||||||
"ournewblend" "$R/blends/newblend/new-blend.blend"
|
|
||||||
)
|
|
||||||
```
|
|
||||||
|
|
||||||
As you can see, the map is a key-value storage. So you can have an alias
|
|
||||||
(name) for your blend, and just use that to point to the path of the
|
|
||||||
blend. The blend file will be sourced by the SDK once it is told to do
|
|
||||||
so.
|
|
||||||
|
|
||||||
|
|
||||||
### A configuration file
|
|
||||||
|
|
||||||
For having a finer-grained control of what goes into our build, we can
|
|
||||||
create a config file for our blend. From here we can easily control
|
|
||||||
any configurable aspect of it, such as packages that go in or out, the
|
|
||||||
blend name, and much more. **Make sure you source this file from your
|
|
||||||
blend.**
|
|
||||||
|
|
||||||
Adding and removing packages was abstractly mentioned earlier: it goes
|
|
||||||
into two separate arrays holding package names. To add packages, we
|
|
||||||
append to the `extra_packages` array, which would look like this:
|
|
||||||
|
|
||||||
```
|
|
||||||
extra_packages+=(
|
|
||||||
my_new_package
|
|
||||||
foo
|
|
||||||
bar
|
|
||||||
baz
|
|
||||||
)
|
|
||||||
```
|
|
||||||
|
|
||||||
This would install the four packages `my_new_package`, `foo`, `bar`,
|
|
||||||
and `baz` along with the ones predefined in either _libdevuansdk_ or
|
|
||||||
the SDK you are using. You may also want to see which those are in
|
|
||||||
case you wish to exclude them, but they are sane and useful utilities
|
|
||||||
which should be included in your build if possible. Overriding all
|
|
||||||
those packages, you would need to reset the whole array, so you would
|
|
||||||
simply issue this:
|
|
||||||
|
|
||||||
```
|
|
||||||
extra_packages=(
|
|
||||||
my_new_package
|
|
||||||
foo
|
|
||||||
bar
|
|
||||||
baz
|
|
||||||
)
|
|
||||||
```
|
|
||||||
|
|
||||||
As you can see, we no longer have the `+=`, but rather only `=`, which
|
|
||||||
means we are not appending to the array, but rather redefining it.
|
|
||||||
|
|
||||||
All of the above applies as well for removing packages, but in this case
|
|
||||||
the array is called `purge_packages`.
|
|
||||||
|
|
||||||
|
|
||||||
#### Custom packages
|
|
||||||
|
|
||||||
If you want to install deb packages that aren't in any repositories, put
|
|
||||||
them in the blend directory and simply add them to another array in the
|
|
||||||
configuration file. The contents of the arrays are the paths to the
|
|
||||||
debs, relative to this configuration file:
|
|
||||||
|
|
||||||
```
|
|
||||||
custom_deb_packages=(
|
|
||||||
yad_0.27.0-1_amd64.deb
|
|
||||||
palemoon_27.2.0~repack-1_amd64.deb
|
|
||||||
)
|
|
||||||
```
|
|
||||||
|
|
||||||
To trigger the installation of these packages, you will need to copy
|
|
||||||
them to `$R/extra/custom_packages`, and then call the
|
|
||||||
`install_custdebs` function somewhere from your blend.
|
|
||||||
|
|
||||||
|
|
||||||
### Custom files
|
|
||||||
|
|
||||||
Any files you want to add to the system to override what's there by
|
|
||||||
default you can add using a *rootfs overlay*. Create a directory
|
|
||||||
inside your blend directory called *rootfs-overlay* and simply put
|
|
||||||
files inside it. The directory structure is absolute to the image we
|
|
||||||
are building. For example what's in "rootfs-overlay/etc/" would end
|
|
||||||
up in the "/etc" of our final image. See _hier(7)_ in the Linux
|
|
||||||
manpages for more explanation on this directory hierarchy.
|
|
||||||
|
|
||||||
If you end up with any files here, to actually copy them, you will need
|
|
||||||
to either run `cp -f` it, or `rsync` the directory if you prefer.
|
|
||||||
|
|
||||||
|
|
||||||
### The .blend file
|
|
||||||
|
|
||||||
We listed a path to the .blend file in our first step. We need to create
|
|
||||||
this file now.
|
|
||||||
|
|
||||||
Start your blend file with the following, so the sdk is aware of the
|
|
||||||
environment:
|
|
||||||
|
|
||||||
```
|
|
||||||
BLENDPATH="${BLENDPATH:-$(dirname $0)}"
|
|
||||||
source $BLENDPATH/config
|
|
||||||
```
|
|
||||||
|
|
||||||
The minimum blend should contain two functions: `blend_preinst` and
|
|
||||||
`blend_postinst`. These functions are called at specific points in the
|
|
||||||
build, where they give the most power: just after bootstrapping the
|
|
||||||
_vanilla_ system, and just before packaging the final build,
|
|
||||||
respectively.
|
|
||||||
|
|
||||||
|
|
||||||
#### blend_preinst
|
|
||||||
|
|
||||||
A preinst function can look like this:
|
|
||||||
|
|
||||||
```
|
|
||||||
blend_preinst() {
|
|
||||||
fn blend_preinst
|
|
||||||
req=(BLENDPATH R)
|
|
||||||
ckreq || return 1
|
|
||||||
|
|
||||||
notice "executing blend preinst"
|
|
||||||
|
|
||||||
add-user "user" "pass"
|
|
||||||
cp -fv "$BLENDPATH"/*.deb "$R/extra/custom-packages" || zerr
|
|
||||||
install-custdebs || zerr
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
As you can see, the pre-install function will add a new user with the
|
|
||||||
credentials `user:pass`, it will copy our custom debs where they can
|
|
||||||
be used, and finally it will trigger their installation.
|
|
||||||
|
|
||||||
The `fn, req, ckreq` part on the top of the function is a safety check
|
|
||||||
for the function that is enabled by _zuper_. It allows us to check if
|
|
||||||
variables are defined when the function is called and fail if it is
|
|
||||||
wrong. You should utilize this as much as possible. The `zerr` calls
|
|
||||||
are used to exit if the function fails.
|
|
||||||
|
|
||||||
|
|
||||||
#### blend_postinst
|
|
||||||
|
|
||||||
A post-install function can look like the following:
|
|
||||||
|
|
||||||
```
|
|
||||||
blend_postinst() {
|
|
||||||
fn blend_postinst
|
|
||||||
req=(BLENDPATH strapdir)
|
|
||||||
ckreq || return 1
|
|
||||||
|
|
||||||
notice "executing blend postinst"
|
|
||||||
|
|
||||||
sudo cp -vf "$BLENDPATH"/rootfs-overlay/* $strapdir || zerr
|
|
||||||
|
|
||||||
blend_finalize || zerr
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
This function would copy the `rootfs-overlay` to the `strapdir` (which
|
|
||||||
holds our image's filesystem) and it would call the `blend_finalize`
|
|
||||||
function. By default this function doesn't exist, we quote it as an
|
|
||||||
example for you to see how it is possible to call your own functions
|
|
||||||
as well. You can define them within the blend file.
|
|
||||||
|
|
||||||
|
|
||||||
Using a blend
|
|
||||||
-------------
|
|
||||||
|
|
||||||
As previously explained, you can use your blends through the SDK's
|
|
||||||
interactive shell. In _decode-os_ the blend is placed in the root of
|
|
||||||
the git repository, and the sdk wrappers are located within. Therefore
|
|
||||||
an SDK would have to source it with such a path:
|
|
||||||
|
|
||||||
```
|
|
||||||
$R/../decode.blend
|
|
||||||
```
|
|
||||||
|
|
||||||
If you take a look at _vm-sdk_'s `sdk` file, you will see the
|
|
||||||
`blend_map` array. Using a new blend requires you to add it to this
|
|
||||||
map in the same manner. The map is key-value formatted, and on the
|
|
||||||
left you have an alias of your blend, and on the right you have a
|
|
||||||
script you have to write. It can either be the blend itself or any
|
|
||||||
helper file you might need to initialize your blend.
|
|
||||||
|
|
||||||
After you've added it to the blend map, you simply initialize the SDK,
|
|
||||||
and use the same *load* command we learned earlier, while appending
|
|
||||||
the blend alias and any optional argument.
|
|
||||||
|
|
||||||
```
|
|
||||||
$ zsh -f
|
|
||||||
$ source sdk
|
|
||||||
$ load devuan decode <these> <arguments> <we> <can> <use> <in> <the> <blend>
|
|
||||||
```
|
|
||||||
|
|
||||||
With this, we've initialized our *decode* blend. It's always good to add a
|
|
||||||
*notice()* call to your blend to signal it's been loaded successfully.
|
|
||||||
|
|
||||||
Once this is done, we simply build the image the same way we have
|
|
||||||
learned before:
|
|
||||||
|
|
||||||
```
|
|
||||||
$ build_vagrant_dist
|
|
||||||
```
|
|
||||||
|
|
||||||
Consult the _vm-sdk_ chapter for this.
|
|
||||||
|
|
@ -1,131 +0,0 @@
|
||||||
The Devuan SDK more in-depth
|
|
||||||
============================
|
|
||||||
|
|
||||||
The following parts will explain the Devuan SDK more technically. It
|
|
||||||
will show its configuration, important functions, and show how it all
|
|
||||||
glues together.
|
|
||||||
|
|
||||||
|
|
||||||
Configuration
|
|
||||||
-------------
|
|
||||||
|
|
||||||
Much of the _libdevuansdk_ configuration is done in
|
|
||||||
`libdevuansdk/config`. Here you can edit the defaults if you wish to
|
|
||||||
do something your needs are expressing. However, overriding these
|
|
||||||
through upper levels is recommended.
|
|
||||||
|
|
||||||
|
|
||||||
### `config` file
|
|
||||||
|
|
||||||
`vars` and `arrs` are global arrays for holding other global variables
|
|
||||||
and arrays, respectively. This is required for `zuper` and helps a lot
|
|
||||||
with debugging. If you declare new variables or arrays, add them to the
|
|
||||||
aforementioned variables.
|
|
||||||
|
|
||||||
|
|
||||||
* `os` holds the name of the distribution being worked on.
|
|
||||||
|
|
||||||
* `release` holds the release codename of the distribution. Used for apt
|
|
||||||
repositories mostly.
|
|
||||||
|
|
||||||
* `version` is the version of the distribution being worked on.
|
|
||||||
|
|
||||||
* `mirror` is a mirror holding the required packages for `debootstrap`.
|
|
||||||
|
|
||||||
* `section` are the sections of the repository. For adding in
|
|
||||||
`/etc/apt/sources.list`. Separate them with whitespaces.
|
|
||||||
|
|
||||||
* `image_name` is the output name of the raw image. If you declare a
|
|
||||||
blend or a device name (arm-sdk), they will be appended to this name.
|
|
||||||
|
|
||||||
* `rootcredentials` and `usercredentials` are currently placeholders.
|
|
||||||
|
|
||||||
* `core_packages` is an array holding the core packages that will be
|
|
||||||
installed in the bootstrap process.
|
|
||||||
|
|
||||||
* `base_packages` is an array holding the base packages that will be
|
|
||||||
installed at a later point in the bootstrap process.
|
|
||||||
|
|
||||||
* `purge_packages` is an array of packages that will get purged at the
|
|
||||||
end of the bootstrap process.
|
|
||||||
|
|
||||||
|
|
||||||
Helper functions
|
|
||||||
----------------
|
|
||||||
|
|
||||||
You can find useful helper functions in `libdevuansdk/zlibs/helpers`.
|
|
||||||
They are intended to help when it comes to writing wrappers, as well
|
|
||||||
as making the developers' jobs easier for developing
|
|
||||||
_libdevuansdk_. Some of these functions are required for
|
|
||||||
_libdevuansdk_ to work properly as well.
|
|
||||||
|
|
||||||
|
|
||||||
### `build_image_dist()`
|
|
||||||
|
|
||||||
This function is a kind of a wrapper function. It's used in _arm-sdk_
|
|
||||||
to build a complete dd-able image from start to end. To run, it
|
|
||||||
requires `$arch`, `$size`, `$parted_type`, `$workdir`, `$strapdir`,
|
|
||||||
and `$image_name` to be declared. See the section dedicated to
|
|
||||||
"Creating wrappers" for insight on these variables.
|
|
||||||
|
|
||||||
The workflow of this function is bootstrapping a complete _rootfs_,
|
|
||||||
creating a raw image, installing/compiling a kernel, rsyncing
|
|
||||||
everything to the raw image, and finally compressing the raw image.
|
|
||||||
|
|
||||||
This same workflow is applied in the next two functions in this file,
|
|
||||||
which are `build_iso_dist` and `build_vagrant_dist`. To get a better
|
|
||||||
understanding of _libdevuansdk_, it's recommended to go through one of
|
|
||||||
these functions and following it deeper to find and figure out the
|
|
||||||
other functions and how they work together.
|
|
||||||
|
|
||||||
|
|
||||||
### `devprocsys()`
|
|
||||||
|
|
||||||
This function is a simple helper function that takes two arguments. It
|
|
||||||
mounts or unmounts `/dev`, `/proc`, and `/sys` filesystems to or from
|
|
||||||
wherever you tell it to. For example:
|
|
||||||
|
|
||||||
```
|
|
||||||
$ devprocsys mount $strapdir
|
|
||||||
$ devprocsys umount $strapdir
|
|
||||||
|
|
||||||
```
|
|
||||||
|
|
||||||
It is very necessary to use this if one wants to do anything requiring
|
|
||||||
access to hardware or the system's resources, i.e. cryptography.
|
|
||||||
|
|
||||||
|
|
||||||
### `dpkgdivert()`
|
|
||||||
|
|
||||||
This function, like `devprocsys` takes two arguments and will create
|
|
||||||
or remove a dpkg diversion in the place you tell it to and remove
|
|
||||||
`invoke-rc.d` so that _apt_ does not autostart daemons when they are
|
|
||||||
installed.
|
|
||||||
|
|
||||||
|
|
||||||
### `chroot-script()`
|
|
||||||
|
|
||||||
This very useful functions allows you to _chroot_ into `$strapdir` and
|
|
||||||
execute the script/binary that's passed as a parameter to this
|
|
||||||
function. It also takes an optional argument `-d` that will call
|
|
||||||
`dpkgdivert` on and off before and after execution.
|
|
||||||
|
|
||||||
The `chroot-script` is also an example on its own that shows how to use
|
|
||||||
the `chroot-script` function.
|
|
||||||
|
|
||||||
|
|
||||||
Mandatory variables
|
|
||||||
-------------------
|
|
||||||
|
|
||||||
* `$R` is the root directory of a wrapper. It's defined already in all
|
|
||||||
the existing ones. In almost evert situation it can be `$PWD`.
|
|
||||||
|
|
||||||
* `$workdir` is the working directory of the current build. A sane
|
|
||||||
default is `$R/tmp/workdir`
|
|
||||||
|
|
||||||
* `$strapdir` is the bootstrap directory of the build. It holds the
|
|
||||||
rootfs when you debootstrap it, and customize it further on. Default
|
|
||||||
is `$workdir/rootfs`.
|
|
||||||
|
|
||||||
* `$arch` is the CPU architecture of the build. I.e. `amd64`, `armhf`,
|
|
||||||
etc.
|
|
||||||
|
|
@ -1,45 +0,0 @@
|
||||||
toaster.do
|
|
||||||
==========
|
|
||||||
|
|
||||||
The **toaster.do** setup is a modular web app relying on different
|
|
||||||
parts of DECODE's CI (continuous integration) and operating system
|
|
||||||
development software (SDK) used to facilitate builds of customized
|
|
||||||
Devuan images using Dockerfiles and a web interface. It allows us to
|
|
||||||
have a seamless way of using the Dockerfiles that are used in testing
|
|
||||||
to make production images using the same Dockerfile. This brings a
|
|
||||||
deterministic approach to debugging and allows centralization of
|
|
||||||
resources, while avoiding extra work needed to write a Devuan blend.
|
|
||||||
|
|
||||||
The web application is public on https://toaster.dyne.org
|
|
||||||
|
|
||||||
All following documentation contained in this document details the
|
|
||||||
internals of this application, of the components and infrastructure
|
|
||||||
that it is using. Unless specifically interested in these
|
|
||||||
implementation details, the web application facilitates the adoption
|
|
||||||
of all features described through a simple visual workflow.
|
|
||||||
|
|
||||||
The setup is comprised of a web interface written in Clojure, a backend
|
|
||||||
glue written in Python, the Devuan SDK, and the Jenkins CI system.
|
|
||||||
|
|
||||||
The main repository of this software component is
|
|
||||||
https://github.com/decodeproject/toaster.do
|
|
||||||
|
|
||||||
|
|
||||||
Clojure frontend
|
|
||||||
----------------
|
|
||||||
|
|
||||||
The Clojure frontend is an embedded web server with its own database,
|
|
||||||
which allows for managing of users. A user registered within this part
|
|
||||||
is then allowed to upload Dockerfiles and manage their image builds.
|
|
||||||
|
|
||||||
The frontend talks to the Python backend through SSH, and runs a
|
|
||||||
specific command to enable or disable a build job.
|
|
||||||
|
|
||||||
|
|
||||||
Jenkins backend
|
|
||||||
---------------
|
|
||||||
|
|
||||||
The backend glue is a Python tool which talks to Jenkins itself and
|
|
||||||
does all the managing and configuration of build jobs. It serves as the
|
|
||||||
backend to the Devuan SDK's web interface and is executed by the web CGI
|
|
||||||
when a build function is requested.
|
|
||||||
|
|
@ -1,130 +0,0 @@
|
||||||
Tor DAM
|
|
||||||
=======
|
|
||||||
|
|
||||||
|
|
||||||
Tor Distributed Announce Mechanism (DAM) is a protocol and tooling for
|
|
||||||
mapping machines in the Tor network running this software.
|
|
||||||
|
|
||||||
The Tor DAM network is imagined to be pseudo-distributed inside the Tor
|
|
||||||
network itself. Nodes running Tor DAM can use an existing entrypoint and
|
|
||||||
start announcing themselves to the entry point(s), or they can be their
|
|
||||||
own and let others announce to themselves. Tor DAM will store all of
|
|
||||||
these announcements in a storage backend and utilize it to expand the
|
|
||||||
knowledge of the nodes using this software. Over time the network will
|
|
||||||
keep expanding and the user will be able to see all other nodes in the
|
|
||||||
network either by querying the storage backend, or visualizing it with
|
|
||||||
some kind of software.
|
|
||||||
|
|
||||||
|
|
||||||
Abstract
|
|
||||||
--------
|
|
||||||
|
|
||||||
* Every node has a HTTP API allowing to list other nodes and announce
|
|
||||||
new ones.
|
|
||||||
* They keep propagating to all valid nodes they know.
|
|
||||||
* Announcing implies the need of knowledge of at least one or two nodes.
|
|
||||||
* It is possible to make this random enough once there are at least 6
|
|
||||||
nodes in the network.
|
|
||||||
* A node announces itself to others by sending a JSON-formatted HTTP
|
|
||||||
POST request to one or more active node.
|
|
||||||
* Once the POST request is received, the node will validate the
|
|
||||||
request and return a secret encrypted with the requester's public
|
|
||||||
key.
|
|
||||||
* The requester will try to decrypt this secret, and return the
|
|
||||||
secret in plain text back to the node it's announcing to, along
|
|
||||||
with a cryptographic signature, so the node can confirm the
|
|
||||||
requester is in actual possession of the private key.
|
|
||||||
* Tor DAM **does not validate** if a node is malicious or not. This is a
|
|
||||||
layer that has to be established on top. Tor DAM is just the entry
|
|
||||||
point into the network.
|
|
||||||
|
|
||||||
|
|
||||||
Protocol
|
|
||||||
--------
|
|
||||||
|
|
||||||
A node announcing itself has to do a JSON-formatted HTTP POST request to
|
|
||||||
one or more active nodes with the format explained below. N.B. The
|
|
||||||
strings shown in this document might not be valid, but they represent a
|
|
||||||
correct example.
|
|
||||||
|
|
||||||
* `type` reflects the type of the node
|
|
||||||
* `address` holds the address of the Tor hidden service
|
|
||||||
* `message` is the message that has to be signed using the private key
|
|
||||||
of this same hidden service.
|
|
||||||
* `signature` is the base64 encoded signature of the above message.
|
|
||||||
* `secret` is a string that is used for exchanging messages between the
|
|
||||||
client and server.
|
|
||||||
|
|
||||||
|
|
||||||
```
|
|
||||||
{
|
|
||||||
"type": "node",
|
|
||||||
"address": "22mobp7vrb7a4gt2.onion",
|
|
||||||
"message": "I am a DAM node!",
|
|
||||||
"signature": "BuB/Dv8E44CLzUX88K2Ab0lUNS9A0GSkHPtrFNNWZMihPMWN0ORhwMZBRnMJ8woPO3wSONBvEvaCXA2hvsVrUJTa+hnevQNyQXCRhdTVVuVXEpjyFzkMamxb6InrGqbsGGkEUqGMSr9aaQ85N02MMrM6T6JuyqSSssFg2xuO+P4=",
|
|
||||||
"secret": ""
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Sending this as a POST request to a node will make it ask for the
|
|
||||||
public key of the given address from a "hidden service directory"
|
|
||||||
(HSDir) in the Tor network. It will retrieve the public key and try to
|
|
||||||
validate the signature that was made. Validating this, we assume that
|
|
||||||
the requester is in possession of the private key.
|
|
||||||
|
|
||||||
Following up, the node shall generate a cryptographically secure random
|
|
||||||
string and encrypt it using the before acquired public key. It will then
|
|
||||||
be encoded using base64 and sent back to the client:
|
|
||||||
|
|
||||||
|
|
||||||
```
|
|
||||||
{
|
|
||||||
"secret": "eP07xSZWlDdK4+AL0WUkIA3OnVTc3sEgu4MUqGr43TUXaJLfAILvWxKihPxytumBmdJ4LC45LsrdDuhmUSmZZMJxxiLmB4Gf3zoWa1DmStdc147VsGpexY05jaJUZlbmG0kkTFdPmdcKNbis5xfRn8Duo1e5bOPj41lIopwiil0="
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
The client will try to decode and decrypt this secret, and send it back
|
|
||||||
to the node to complete its part of the handshake. The POST request this
|
|
||||||
time will contain the following data:
|
|
||||||
|
|
||||||
* `type` reflects the type of the node
|
|
||||||
* `address` holds the address of the Tor hidden service
|
|
||||||
* `message` is the decrypted and base64 encoded secret that the server
|
|
||||||
had just sent us.
|
|
||||||
* `signature` is the base64 encoded signature of the above secret.
|
|
||||||
* `secret` is a copy of `message` here.
|
|
||||||
|
|
||||||
|
|
||||||
```
|
|
||||||
{
|
|
||||||
"type": "node",
|
|
||||||
"address": "22mobp7vrb7a4gt2.onion",
|
|
||||||
"message": "ZShhYHYsRGNLOTZ6YUwwP3ZXPnxhQiR9UFVWfmk5TG56TEtLb04vMms+OTIrLlQ7aS4rflR3V041RG5Je0tnYw==",
|
|
||||||
"signature": "L1N+VEi3T3aZaYksAy1+0UMoYn7B3Gapfk0dJzOUxUtUYVhj84TgfYeDnADNYrt5UK9hN/lCTIhsM6zPO7mSjQI43l3dKvMIikqQDwNey/XaokyPI4/oKrMoGQnu8E8UmHmI1pFvwdO5EQQaKbi90qWNj93KB/NlTwqD9Ir4blY=",
|
|
||||||
"secret": "ZShhYHYsRGNLOTZ6YUwwP3ZXPnxhQiR9UFVWfmk5TG56TEtLb04vMms+OTIrLlQ7aS4rflR3V041RG5Je0tnYw=="
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
The node will verify the received plain secret against what it has
|
|
||||||
encrypted to validate. If the comparison yields no errors, we assume
|
|
||||||
that the requester is actually in possession of the private key. If the
|
|
||||||
node is not valid in our database, we will complete the handshake by
|
|
||||||
welcoming the client into the network:
|
|
||||||
|
|
||||||
|
|
||||||
```
|
|
||||||
{
|
|
||||||
"secret": "Welcome to the DAM network!"
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Further on, the node will append useful metadata to the struct. We will
|
|
||||||
add the encoded public key, timestamps of when the client was first seen
|
|
||||||
and last seen, and a field to indicate if the node is valid. The latter
|
|
||||||
is not to be handled by Tor DAM, but rather the upper layer, which
|
|
||||||
actually has consensus handling.
|
|
||||||
|
|
||||||
If the node is valid in another node's database, the remote node will
|
|
||||||
then propagate back all the valid nodes it knows (including itself) back
|
|
||||||
to the client in a gzipped and base64 encoded JSON struct. The client
|
|
||||||
will then handle this and update its own database accordingly.
|
|
||||||
|
|
@ -1,27 +0,0 @@
|
||||||
#!/bin/sh
|
|
||||||
#
|
|
||||||
# This script will run the keygen script for a requested amount of times.
|
|
||||||
# It takes an optional integer parameter - amount - for the amount of dockers.
|
|
||||||
# Otherwise it will default to 5.
|
|
||||||
|
|
||||||
usage() {
|
|
||||||
echo "$(basename $0) [number]"
|
|
||||||
exit 1
|
|
||||||
}
|
|
||||||
|
|
||||||
[ -z "$1" ] && AMOUNT=5
|
|
||||||
|
|
||||||
case "$1" in
|
|
||||||
*[!0-9]*)
|
|
||||||
usage
|
|
||||||
;;
|
|
||||||
*)
|
|
||||||
AMOUNT="$1"
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
|
|
||||||
rm -f onions.txt
|
|
||||||
|
|
||||||
for i in $(seq 1 $AMOUNT); do
|
|
||||||
./keygen
|
|
||||||
done
|
|
||||||
|
|
@ -1,9 +0,0 @@
|
||||||
#!/bin/sh
|
|
||||||
#
|
|
||||||
# This script will start containers that were generated with create.sh
|
|
||||||
|
|
||||||
for i in $(cat onions.txt); do
|
|
||||||
onion="$(echo $i | cut -d':' -f2)"
|
|
||||||
container="$(docker run -d dyne/decodeos:$onion)"
|
|
||||||
echo "Started container $container for $onion"
|
|
||||||
done
|
|
||||||
|
|
@ -1,12 +0,0 @@
|
||||||
#!/bin/sh
|
|
||||||
#
|
|
||||||
# This script will stop and delete the created containers and images.
|
|
||||||
|
|
||||||
containers="$(docker container ls | awk '/dyne\/decodeos:.*\.onion/ {print $1}')"
|
|
||||||
|
|
||||||
echo "$containers" | xargs docker stop
|
|
||||||
echo "$containers" | xargs docker rm
|
|
||||||
|
|
||||||
images="$(docker images | awk '/dyne\/decodeos:.*\.onion/ {print $3}')"
|
|
||||||
|
|
||||||
echo "$images" | xargs docker rmi
|
|
||||||
|
|
@ -6,7 +6,6 @@ onion=$(docker exec $container dam-client -gen 2>&1| awk '/Our hostname/ {print
|
||||||
echo "New DECODE-OS node address: $onion"
|
echo "New DECODE-OS node address: $onion"
|
||||||
image=$(docker commit $container dyne/decodeos:$onion)
|
image=$(docker commit $container dyne/decodeos:$onion)
|
||||||
echo "Docker image: dyne/decodeos:$onion"
|
echo "Docker image: dyne/decodeos:$onion"
|
||||||
echo "DIR:$onion" >> onions.txt
|
|
||||||
echo "$image"
|
echo "$image"
|
||||||
container=$(docker stop $container)
|
container=$(docker stop $container)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -27,16 +27,12 @@ prompt=DECODE
|
||||||
command=tor
|
command=tor
|
||||||
|
|
||||||
[program:dam-dir]
|
[program:dam-dir]
|
||||||
command=dam-dir -t -ttl 10
|
command=dam-dir
|
||||||
redirect_stderr=true
|
redirect_stderr=true
|
||||||
stdout_logfile=/var/log/dam-dir.log
|
|
||||||
stderr_logfile=/var/log/dam-dir.err
|
|
||||||
|
|
||||||
[program:dam-client]
|
[program:dam-client]
|
||||||
command=dam-client -ai 5 -dh https://dam.decodeproject.eu/testnet.txt
|
command=dam-client
|
||||||
redirect_stderr=true
|
redirect_stderr=true
|
||||||
stdout_logfile=/var/log/dam-client.log
|
|
||||||
stderr_logfile=/var/log/dam-client.err
|
|
||||||
|
|
||||||
[program:redis-commander]
|
[program:redis-commander]
|
||||||
command=redis-commander
|
command=redis-commander
|
||||||
|
|
|
||||||
2
live-sdk
2
live-sdk
|
|
@ -1 +1 @@
|
||||||
Subproject commit aff7daadab16657b69ade1a01315389f25bb032d
|
Subproject commit dd708cfcfeda609c0b33fc61882a9b7deff3f6dd
|
||||||
2
vm-sdk
2
vm-sdk
|
|
@ -1 +1 @@
|
||||||
Subproject commit 742bd8318604acf2ebf45594eda053c3ea78e9f6
|
Subproject commit 2826c9255813b56a0f7cb0b43d4dca5fd6914ef3
|
||||||
Loading…
Reference in New Issue