From 579f27675cd921a12833462e2d6dbd5318df310f Mon Sep 17 00:00:00 2001 From: Jaromil Date: Mon, 12 Nov 2018 11:03:48 +0100 Subject: [PATCH] initial commit after completion --- config.zsh | 36 +++++ introduction.md | 55 +++++++ sdk-01-overview.md | 293 +++++++++++++++++++++++++++++++++++++ sdk-02-blends.md | 322 +++++++++++++++++++++++++++++++++++++++++ sdk-03-technical.md | 131 +++++++++++++++++ toaster-01-overview.md | 45 ++++++ tordam-01-overview.md | 130 +++++++++++++++++ views/abstract.txt | 10 ++ views/index.txt | 6 + views/template.tex | 177 ++++++++++++++++++++++ 10 files changed, 1205 insertions(+) create mode 100644 config.zsh create mode 100644 introduction.md create mode 100644 sdk-01-overview.md create mode 100644 sdk-02-blends.md create mode 100644 sdk-03-technical.md create mode 100644 toaster-01-overview.md create mode 100644 tordam-01-overview.md create mode 100644 views/abstract.txt create mode 100644 views/index.txt create mode 100644 views/template.tex diff --git a/config.zsh b/config.zsh new file mode 100644 index 0000000..2c581d4 --- /dev/null +++ b/config.zsh @@ -0,0 +1,36 @@ +# Writedown +# generic configuration defaults for rendered files +# to render, put the writedown directory here +# see: https://github.com/dyne/writedown + +WRITEDOWN_TITLE="D4.11 Stable DECODE OS release" +WRITEDOWN_AUTHOR="Ivan Jelinčić, Denis Roio" +WRITEDOWN_AFFILIATION="Dyne.org Foundation" +WRITEDOWN_DATE="22 October 2017" +WRITEDOWN_TAGS="[software, integration, decentralized, os, distributed, computing, linux, distro, microservice, tor]" + +# number for each section +WRITEDOWN_NRSEC=yes +# table of contents +WRITEDOWN_TOC=yes +# bibliographic citation style (see writedown/citstyle) +WRITEDOWN_CITSTYLE=harvard-kings-college-london +# font size +WRITEDOWN_FONTSIZE=14pt +# bibtex file for bibliographic sources +# WRITEDOWN_BIB=./views/references.bib +# latex template (header and footer) +#WRITEDOWN_LATEX_TEMPLATE=/usr/share/pandoc/data/templates/latex.tex + +# different formats as supported by pandoc. +# to activate uncomment and fill, then use dash (-) as first argument +# i.e: ./writedown/render - +# WRITEDOWN_OUTPUT_FORMAT=epub +# WRITEDOWN_OUTPUT_EXTENSION=epub + +# default pandoc base command +# WRITEDOWN_PANDOC="pandoc --smart --standalone -f markdown + +# Experimental features are commented below + +# WRITEDOWN_ZOTERO="no" diff --git a/introduction.md b/introduction.md new file mode 100644 index 0000000..b47f2c2 --- /dev/null +++ b/introduction.md @@ -0,0 +1,55 @@ +Stable DECODE OS release +======================== + +This document accompanies the stable release of the DECODE OS, one of +the core development outputs of the DECODE project, aimed at providing +a reliable operating system to run application space development in an +environment ensuring privacy by design outside of the application +domain. This deliverable references, without duplication of +information, the research and development done and detailed in +previous deliverables D4.1 and D4.4. + +The DECODE OS is a GNU+Linux distribution based on Devuan.org to +provide a minimalist base for distributed computing micro-services +capable of targeting any mainstream hardware platform, from +virtual-machines to ARM boards to bare-metal server racks. + +The main website for this distribution is https://decodeos.dyne.org + + +As part of the DECODE OS distribution, backend software applications +have been developed to implement + +1. a front-end web application to facilitate the adoption of the + DECODE continuous integration infrastructure (toaster) + https://toaster.dyne.org +2. a continuous integration system to release and customize new + versions of DECODE OS (SDK) https://git.devuan.org/sdk +3. a private peer-to-peer network over the Tor protocol (tor-dam) + https://github.com/decodeproject/tor-dam + +These core features of these three components will be described in the +following sections of this document, along with operational +instructions. + +Due to the experimental stage of development of other components in +DECODE and according to the LEAN principles declared in the project, +this stable release doesn't only constitute a final point of arrival +for this development task. What DECODE OS can do today is facilitating +the deployment of lab-tested software applications (for example made +in a Docker format, widely adopted by other partners in DECODE) and +render these prototypes into a production ready format that can be +deployed on the open-hardware DECODE BOX as well on virtual-machines. + +We consider this achievement highly beneficial for a project whose +development is still in-flux, as well for the free and open source +community out there, since the access to the powerful features of the +SDK is now made very easy via an integrated continuous pipeline. + +In light of these advantages, there is a clear intention within our +organisation (mainly by DYNE) to keep maintaining DECODE OS also +beyond the span of the project and this very task now concluded, since +it greatly helps the manning of prototypes into stable production +environments. + + diff --git a/sdk-01-overview.md b/sdk-01-overview.md new file mode 100644 index 0000000..c0f075d --- /dev/null +++ b/sdk-01-overview.md @@ -0,0 +1,293 @@ +The DECODE SDK +============== + +The DECODE SDK is a unique build framework written to ease maintenance +and production of various types of the Devuan distribution images, +such as: live ISOs, virtual machine images, and images targeted at +embedded ARM boards. This section explains how to use the SDK, gives +and inside look at its various parts and documents the workflow to be +used when modifying its code. + +The SDK is designed in such a way that there are levels of priority +within the scripts. First there is `libdevuansdk`, which holds the +vanilla configuration, then come the various wrappers targeted around +specific targets (`live`, `virtual`, `embedded`), and afterwards we +optionally add more on top of it if we need to customize or override +specific functions. This is for example the case with DECODE OS, +where we have to add additional software and extra components on top +of the base Devuan system. + + +libdevuansdk +------------ + +_libdevuansdk_ is the core of any part of the Devuan SDK. It holds the +common knowledge between all of the upper wrappers such as _live-sdk_, +_vm-sdk_, and _arm-sdk_. Simply put, it is a shell script library to +unify the use and creation of various functions spread throughout the +complete Devuan SDK. + +The wrappers are designed to be used interactively from a terminal, as +well as automated from shell scripts. _libdevuansdk_ uses an +additional _zsh_ library called [zuper](https://github.com/dyne/zuper) +to ease the variable declaration and scoping, as well as error +checking and debugging. However, _zuper_ is not included in +_libdevuansdk_ itself - one is required to include it in its +respective wrapper. _live-sdk_, _vm-sdk_, and _arm-sdk_ can be taken +as example. libdevuansdk itself has some software dependencies that +should be installed prior to use: + +``` +zsh +debootstrap +sudo +kpartx +cgpt +xz-utils +``` + + +### Workflow + +Working with _libdevuansdk_ splits into categories of what you want to +do. _zlibs_ are files separated into the following categories: + +* ***bootstrap*** Contains the functions for the bootstrap process. + Creating a minimal debootstrap base system, and making it into a + compressed file (tar.gz) for later use so one does not have to wait + for the lengthy bootstrap process on each consequent build. + +* ***helpers*** Contains the helper functions for _libdevuansdk_ that + make the workflow a bit easier to use and handle. + +* ***imaging*** Contains the functions necessary for creating raw + dd-able images. + +* ***rsync*** Contains rsync and file copying functions. + +* ***sysconf*** Contains the default system configuration. + + +### Usage + +As libdevuansdk is not very useful when invoked on its own, its usage +will be explained at later parts, for each specific wrapper. The +technical documentation of _libdevuansdk_ will follow in its +appropriate section. + + +The wrappers +------------ + +As mentioned, _libdevuansdk_ is the core library we wrap around. The +currently existing wrappers are called _live-sdk_, _vm-sdk_, and +_arm-sdk_. These facilitate the builds of liveCDs, virtual machines, and +images for embedded ARM devices, respectively. Each of them have their +own section in this paper. + +Since all of these wrappers, along with _libdevuansdk_, hold a +_vanilla_ Devuan configuration, it is best to keep their code +untouched. To allow for custom configurations, we introduced a concept +called *blends*. Blends are a simple way to customize the base image +of the OS-to-be before building it, allowing to easily add packages, +kernels, and virtually anything one might want to do in the +image. This exactly is the case with DECODE OS. + + +arm-sdk +------- + +The _arm-sdk_ is our way of facilitating builds for embedded ARM boards +such as Allwinner-based CPUs, Raspberry Pis, Chromebooks, etc. It holds +a knowledgebase for a number of embedded devices, and how to build +according kernels and bootloaders. + + +### Directory structure + +_arm-sdk_'s directory structure is separated into places where we hold +our boards and their kernel configurations, device-specific +directories with firmware and/or configuration, and a _lib_ directory +(where we keep _libdevuansdk_ and the like). + + +### Obtaining arm-sdk + +The SDK, like any other part of Devuan's software toolchain, should be +obtained via _git_. The repositories are hosted on Devuan's Gitlab. To +grab it, we simply issue a _git clone_ command on a terminal, and +since it contains linked git submodules - we append _--recursive_ to +it: + +``` +$ git clone https://git.devuan.org/sdk/arm-sdk --recursive +``` + +Consult the _README.md_ file found in this repository to see what are +the required dependencies to use _arm-sdk_. + + +### Using arm-sdk + +Once the build system is obtained, it can now be used interactively. The +process is very simple, and to build an image one can actually use a +single shell command. However, we shall first show how it works. + +In _arm-sdk_, every board has its own script located in the _boards_ +directory. In most cases, these scripts contain functions to build the +Linux kernel, and a bootloader needed for the board to boot. This is +the only difference between all the boards, which requires every board +to have their own script. We are able to reuse the _rootfs_ that was +bootstrapped before. For our example, let's take the _Nokia N900_ +build script. To build a _vanilla_ image for it, we simply issue: + + +``` +$ zsh -f -c 'source sdk && load devuan n900 && build_image_dist' + +``` + +This will fire up the build process, and after a certain amount of time +we will have our compressed image ready and checksummed inside the +_dist_ directory. + +The oneliner above is self-explanatory: We first start a new untainted +shell, source the sdk file to get an interactive SDK shell, then we +initialize the operating system along with the board we are building, +and finally we issue a helper command that calls all the necessary +functions to build our image. The _load_ command takes an optional +third argument which is the name of our _blend_ (the way to customize +our _vanilla_ image) which will be explained later. So in this case, +our oneliner would look like: + +``` +$ zsh -f -c 'source sdk && load devuan n900 decode && build_image_dist' +``` + +This would create an image with the _"decode"_ blend, which is available +by cloning the DECODE OS git repository. The *build_image_dist* command +is a helper function located in _libdevuansdk_ that wraps around the 8 +functions needed to build our image. They are all explained in the +technical part of this paper. + + +live-sdk +-------- + +The _live-sdk_ is used to build bootable images, better known as Live +CDs. Its structure is very similar to _vm-sdk_ and is a lot smaller than +_arm-sdk_. + + +### Directory structure + +Unlike _arm-sdk_, in _live-sdk_ we have no need for specific boards or +setups, so in this case we only host the interactive shell init, and +libraries. + + +### Obtaining live-sdk + +The SDK, like any other, should be obtained via _git_. The repositories +are hosted on Devuan's Gitlab. To grab it, we simply issue a _git clone_ +command, an since it contains git submodules - we append _--recursive_ +to it: + +``` +$ git clone https://git.devuan.org/sdk/live-sdk --recursive +``` + +Consult the _README.md_ file found in this repository to see what are +the required dependencies to use _live-sdk_. + + +### Using live-sdk + +Much like _arm-sdk_, the _live-sdk_ is used the same way. With two +specific differences. Since we don't have any need for specific +boards, when loading we don't specify a board, but rather the CPU +architecture we are building for. Currently supported are *i386* and +*amd64* which represent 32bit and 64bit respectively. To build a +_vanilla_ live ISO, we issue: + +``` +$ zsh -f -c 'source sdk && load devuan amd64 && build_iso_dist' +``` + +This will start the build process, and after a certain amount of time we +will have our ISO ready and inside the _dist_ directory. + +Just like in _arm-sdk_, we can use a _blend_ and customize our OS: + +``` +$ zsh -f -c 'source sdk && load devuan amd64 decode && build_iso_dist' +``` + +So this would create a live ISO of DECODE OS. Again as noted, this can +be obtained by recursively cloning the corresponding (DECODE-OS) git +repository. + +The *build_iso_dist* command is a helper function located in +_libdevuansdk_ that wraps around the 9 functions needed to build our +image. They are all explained in the technical part of this manual. + + +vm-sdk +------ + +The _vm-sdk_ is used to build VirtualBox/Vagrant boxes, and virtual +images for emulation, in QCOW2 format, which is the byproduct of +building a Vagrant box. Its structure is very similar to _live-sdk_ +and is the smallest of the three wrappers currently found in the +Devuan SDK. + + +### Directory structure + +Like with _live-sdk_, in _vm-sdk_ we have no need for specific boards +or setups, so in this case we only host the interactive shell init, +and libraries. + + +### Obtaining vm-sdk + +The SDK, like any other, should be obtained via _git_. The +repositories are hosted on Devuan's Gitlab. To grab it, we simply +issue a _git clone_ command, an since it contains git submodules - we +append _--recursive_ to it: + +``` +$ git clone https://git.devuan.org/sdk/vm-sdk --recursive +``` + +Consult the _README.md_ file found in this repository to see what are +the required dependencies to use _vm-sdk_. + + +### Using vm-sdk + +Once obtained, we can use it interactively. The process is very simple, +and to build an image we use the oneliner we've already seen above. + +Also like with _live-sdk_, we don't build for specific boards, however +we also do not create any non-amd64 images, so we don't have to pass +an architecture to the load command either. To build a _vanilla_ +Vagrant Box, VirtualBox image, QCOW2 image, and a cloud-based QCOW2 +image, we issue: + +``` +$ zsh -f -c 'source sdk && load devuan && build_vagrant_dist' +``` + +This line would create all the four types of the VM image. + +As shown with the previous two wrappers, the _blend_ concept works as +advertised here as well: + +``` +$ zsh -f -c 'source sdk && load deuvan decode && build_vagrant_dist' +``` + +The *build_vagrant_dist* command is a helper function located in +_libdevuansdk_ that wraps around the 11 functions needed to build our +image. They are all explained in the technical part of this manual. diff --git a/sdk-02-blends.md b/sdk-02-blends.md new file mode 100644 index 0000000..40cb341 --- /dev/null +++ b/sdk-02-blends.md @@ -0,0 +1,322 @@ +Blends +====== + + +Introduction +------------ + +In the Devuan SDK, a _blend_ is the preferred way we use to make +customizations to the _vanilla_ image. Using blends we can very easily +create different flavors of our image, by easily including/excluding +certain software packages, files, or anything we wish to do. Blends +can become a very quick way of creating entire new derivatives of the +original _vanilla_ distribution we are building. + +This time, we will take the DECODE OS as a _blend_ example. In DECODE +OS we provide a blend called _decode_ which is the blend we use to +create a production release of DECODE OS. The blend's files are +contained within their own directory in the _decode-os_ git +repository. + + +Configuration +------------- + +Any SDK requires a single file to act as a _blend_. This file is also a +_zsh_ script, and, at the very least, it must contain two functions +called: + +``` +blend_preinst() +blend_postinst() +``` + +These functions are your pathway to expanding your blend into whatever +you would like it to do. The _preinst_ function is usually called +right after bootstrapping the _vanilla_ root filesystem, and the +_postinst_ function is called near the very end, just before packing +or compressing the image. These two strategic places should be enough +to do changes within the image. If this is not enough, blends also +allow you to simply **override any variable or function** contained +within _libdevuansdk_ or the sdk you are using. + +Our _decode_ blend is such an example. It is a somewhat expanded blend, +not contained within a single file, but rather a directory. This allows +easier maintenance and makes the scripts clearer and cleaner. + + +### Adding and removing packages + +When we want to add or remove specific packages to our build, we have +to override or append to _libdevuansdk_'s arrays. The array for +packages we want installed is called *extra_packages*, and the array +for packages we want purged is called *purge_packages*. In the Decode +blend, these can be found in the _config_ file located inside the +_decode-os_ blend directory. Keep in mind that these arrays could +already contain specific packages, so you are advised to rather append +to them, than overriding them. + +If the packages you want to install are not available in the +repositories, you still have a way of automatically installing +them. All you have to do is copy your corresponding .deb files to the +following directory of the blend: + +``` +$R/extra/custom-packages/ +``` + +And when that is done, just call the function *install-custdebs* + + +Creating a blend +---------------- + +Rather than explaining the following in theory, you are best off +viewing the blend files that are provided with _decode-os_. It is a +fairly simple blend and should give you enough insight on how to +create your own blend. Here are some important guidelines for creating +a blend: + + +* The blend should always contain at least two functions + +This means you must provide *blend_preinst* and *blend_postinst* in your +blend. They don't even have to do anything, but they should be there. +These two functions open the path for you to call any other functions +you created for your blend. + + +* When overriding functions, make sure they provide a result that + doesn't break the API + +Breaking the API may result in unwanted behavior. You should always +study well the functions you are planning to override and figure out if +it is safe to override them in the way you want. The same goes for any +variables as well. + + +* Any arguments used after the blend name when loading from the SDK are + free for you to use in the blend. + +This means you can use anything after the fourth argument (**$4** in +_zsh_) inside your blend if you require passing arguments to it. + +These are some of the more important guidelines. There is plenty more +tricks and quirks, but it's easy to find out how to tweak the +configuration files and the blend in general once you read through a +blend or two on your own. + + +### Enable the blend + +To use your blend in the first place, you need to make the SDK know +about it. Thus you should append the path to your new blend inside +the **blend_map** of the _sdk_ file: + +``` +blend_map=( + "devuan-live" "$R/blends/devuan-live/devuan-live.blend" + "decode" "$R/../decode.blend" + "heads" "$R/../heads.blend" + "ournewblend" "$R/blends/newblend/new-blend.blend" +) +``` + +As you can see, the map is a key-value storage. So you can have an alias +(name) for your blend, and just use that to point to the path of the +blend. The blend file will be sourced by the SDK once it is told to do +so. + + +### A configuration file + +For having a finer-grained control of what goes into our build, we can +create a config file for our blend. From here we can easily control +any configurable aspect of it, such as packages that go in or out, the +blend name, and much more. **Make sure you source this file from your +blend.** + +Adding and removing packages was abstractly mentioned earlier: it goes +into two separate arrays holding package names. To add packages, we +append to the `extra_packages` array, which would look like this: + +``` +extra_packages+=( + my_new_package + foo + bar + baz +) +``` + +This would install the four packages `my_new_package`, `foo`, `bar`, +and `baz` along with the ones predefined in either _libdevuansdk_ or +the SDK you are using. You may also want to see which those are in +case you wish to exclude them, but they are sane and useful utilities +which should be included in your build if possible. Overriding all +those packages, you would need to reset the whole array, so you would +simply issue this: + +``` +extra_packages=( + my_new_package + foo + bar + baz +) +``` + +As you can see, we no longer have the `+=`, but rather only `=`, which +means we are not appending to the array, but rather redefining it. + +All of the above applies as well for removing packages, but in this case +the array is called `purge_packages`. + + +#### Custom packages + +If you want to install deb packages that aren't in any repositories, put +them in the blend directory and simply add them to another array in the +configuration file. The contents of the arrays are the paths to the +debs, relative to this configuration file: + +``` +custom_deb_packages=( + yad_0.27.0-1_amd64.deb + palemoon_27.2.0~repack-1_amd64.deb +) +``` + +To trigger the installation of these packages, you will need to copy +them to `$R/extra/custom_packages`, and then call the +`install_custdebs` function somewhere from your blend. + + +### Custom files + +Any files you want to add to the system to override what's there by +default you can add using a *rootfs overlay*. Create a directory +inside your blend directory called *rootfs-overlay* and simply put +files inside it. The directory structure is absolute to the image we +are building. For example what's in "rootfs-overlay/etc/" would end +up in the "/etc" of our final image. See _hier(7)_ in the Linux +manpages for more explanation on this directory hierarchy. + +If you end up with any files here, to actually copy them, you will need +to either run `cp -f` it, or `rsync` the directory if you prefer. + + +### The .blend file + +We listed a path to the .blend file in our first step. We need to create +this file now. + +Start your blend file with the following, so the sdk is aware of the +environment: + +``` +BLENDPATH="${BLENDPATH:-$(dirname $0)}" +source $BLENDPATH/config +``` + +The minimum blend should contain two functions: `blend_preinst` and +`blend_postinst`. These functions are called at specific points in the +build, where they give the most power: just after bootstrapping the +_vanilla_ system, and just before packaging the final build, +respectively. + + +#### blend_preinst + +A preinst function can look like this: + +``` +blend_preinst() { + fn blend_preinst + req=(BLENDPATH R) + ckreq || return 1 + + notice "executing blend preinst" + + add-user "user" "pass" + cp -fv "$BLENDPATH"/*.deb "$R/extra/custom-packages" || zerr + install-custdebs || zerr +} +``` + +As you can see, the pre-install function will add a new user with the +credentials `user:pass`, it will copy our custom debs where they can +be used, and finally it will trigger their installation. + +The `fn, req, ckreq` part on the top of the function is a safety check +for the function that is enabled by _zuper_. It allows us to check if +variables are defined when the function is called and fail if it is +wrong. You should utilize this as much as possible. The `zerr` calls +are used to exit if the function fails. + + +#### blend_postinst + +A post-install function can look like the following: + +``` +blend_postinst() { + fn blend_postinst + req=(BLENDPATH strapdir) + ckreq || return 1 + + notice "executing blend postinst" + + sudo cp -vf "$BLENDPATH"/rootfs-overlay/* $strapdir || zerr + + blend_finalize || zerr +} +``` + +This function would copy the `rootfs-overlay` to the `strapdir` (which +holds our image's filesystem) and it would call the `blend_finalize` +function. By default this function doesn't exist, we quote it as an +example for you to see how it is possible to call your own functions +as well. You can define them within the blend file. + + +Using a blend +------------- + +As previously explained, you can use your blends through the SDK's +interactive shell. In _decode-os_ the blend is placed in the root of +the git repository, and the sdk wrappers are located within. Therefore +an SDK would have to source it with such a path: + +``` +$R/../decode.blend +``` + +If you take a look at _vm-sdk_'s `sdk` file, you will see the +`blend_map` array. Using a new blend requires you to add it to this +map in the same manner. The map is key-value formatted, and on the +left you have an alias of your blend, and on the right you have a +script you have to write. It can either be the blend itself or any +helper file you might need to initialize your blend. + +After you've added it to the blend map, you simply initialize the SDK, +and use the same *load* command we learned earlier, while appending +the blend alias and any optional argument. + +``` +$ zsh -f +$ source sdk +$ load devuan decode +``` + +With this, we've initialized our *decode* blend. It's always good to add a +*notice()* call to your blend to signal it's been loaded successfully. + +Once this is done, we simply build the image the same way we have +learned before: + +``` +$ build_vagrant_dist +``` + +Consult the _vm-sdk_ chapter for this. diff --git a/sdk-03-technical.md b/sdk-03-technical.md new file mode 100644 index 0000000..e39b502 --- /dev/null +++ b/sdk-03-technical.md @@ -0,0 +1,131 @@ +The Devuan SDK more in-depth +============================ + +The following parts will explain the Devuan SDK more technically. It +will show its configuration, important functions, and show how it all +glues together. + + +Configuration +------------- + +Much of the _libdevuansdk_ configuration is done in +`libdevuansdk/config`. Here you can edit the defaults if you wish to +do something your needs are expressing. However, overriding these +through upper levels is recommended. + + +### `config` file + +`vars` and `arrs` are global arrays for holding other global variables +and arrays, respectively. This is required for `zuper` and helps a lot +with debugging. If you declare new variables or arrays, add them to the +aforementioned variables. + + +* `os` holds the name of the distribution being worked on. + +* `release` holds the release codename of the distribution. Used for apt + repositories mostly. + +* `version` is the version of the distribution being worked on. + +* `mirror` is a mirror holding the required packages for `debootstrap`. + +* `section` are the sections of the repository. For adding in + `/etc/apt/sources.list`. Separate them with whitespaces. + +* `image_name` is the output name of the raw image. If you declare a + blend or a device name (arm-sdk), they will be appended to this name. + +* `rootcredentials` and `usercredentials` are currently placeholders. + +* `core_packages` is an array holding the core packages that will be + installed in the bootstrap process. + +* `base_packages` is an array holding the base packages that will be + installed at a later point in the bootstrap process. + +* `purge_packages` is an array of packages that will get purged at the + end of the bootstrap process. + + +Helper functions +---------------- + +You can find useful helper functions in `libdevuansdk/zlibs/helpers`. +They are intended to help when it comes to writing wrappers, as well +as making the developers' jobs easier for developing +_libdevuansdk_. Some of these functions are required for +_libdevuansdk_ to work properly as well. + + +### `build_image_dist()` + +This function is a kind of a wrapper function. It's used in _arm-sdk_ +to build a complete dd-able image from start to end. To run, it +requires `$arch`, `$size`, `$parted_type`, `$workdir`, `$strapdir`, +and `$image_name` to be declared. See the section dedicated to +"Creating wrappers" for insight on these variables. + +The workflow of this function is bootstrapping a complete _rootfs_, +creating a raw image, installing/compiling a kernel, rsyncing +everything to the raw image, and finally compressing the raw image. + +This same workflow is applied in the next two functions in this file, +which are `build_iso_dist` and `build_vagrant_dist`. To get a better +understanding of _libdevuansdk_, it's recommended to go through one of +these functions and following it deeper to find and figure out the +other functions and how they work together. + + +### `devprocsys()` + +This function is a simple helper function that takes two arguments. It +mounts or unmounts `/dev`, `/proc`, and `/sys` filesystems to or from +wherever you tell it to. For example: + +``` +$ devprocsys mount $strapdir +$ devprocsys umount $strapdir + +``` + +It is very necessary to use this if one wants to do anything requiring +access to hardware or the system's resources, i.e. cryptography. + + +### `dpkgdivert()` + +This function, like `devprocsys` takes two arguments and will create +or remove a dpkg diversion in the place you tell it to and remove +`invoke-rc.d` so that _apt_ does not autostart daemons when they are +installed. + + +### `chroot-script()` + +This very useful functions allows you to _chroot_ into `$strapdir` and +execute the script/binary that's passed as a parameter to this +function. It also takes an optional argument `-d` that will call +`dpkgdivert` on and off before and after execution. + +The `chroot-script` is also an example on its own that shows how to use +the `chroot-script` function. + + +Mandatory variables +------------------- + +* `$R` is the root directory of a wrapper. It's defined already in all + the existing ones. In almost evert situation it can be `$PWD`. + +* `$workdir` is the working directory of the current build. A sane + default is `$R/tmp/workdir` + +* `$strapdir` is the bootstrap directory of the build. It holds the + rootfs when you debootstrap it, and customize it further on. Default + is `$workdir/rootfs`. + +* `$arch` is the CPU architecture of the build. I.e. `amd64`, `armhf`, + etc. diff --git a/toaster-01-overview.md b/toaster-01-overview.md new file mode 100644 index 0000000..351a3a3 --- /dev/null +++ b/toaster-01-overview.md @@ -0,0 +1,45 @@ +toaster.do +========== + +The **toaster.do** setup is a modular web app relying on different +parts of DECODE's CI (continuous integration) and operating system +development software (SDK) used to facilitate builds of customized +Devuan images using Dockerfiles and a web interface. It allows us to +have a seamless way of using the Dockerfiles that are used in testing +to make production images using the same Dockerfile. This brings a +deterministic approach to debugging and allows centralization of +resources, while avoiding extra work needed to write a Devuan blend. + +The web application is public on https://toaster.dyne.org + +All following documentation contained in this document details the +internals of this application, of the components and infrastructure +that it is using. Unless specifically interested in these +implementation details, the web application facilitates the adoption +of all features described through a simple visual workflow. + +The setup is comprised of a web interface written in Clojure, a backend +glue written in Python, the Devuan SDK, and the Jenkins CI system. + +The main repository of this software component is +https://github.com/decodeproject/toaster.do + + +Clojure frontend +---------------- + +The Clojure frontend is an embedded web server with its own database, +which allows for managing of users. A user registered within this part +is then allowed to upload Dockerfiles and manage their image builds. + +The frontend talks to the Python backend through SSH, and runs a +specific command to enable or disable a build job. + + +Jenkins backend +--------------- + +The backend glue is a Python tool which talks to Jenkins itself and +does all the managing and configuration of build jobs. It serves as the +backend to the Devuan SDK's web interface and is executed by the web CGI +when a build function is requested. diff --git a/tordam-01-overview.md b/tordam-01-overview.md new file mode 100644 index 0000000..871e9e5 --- /dev/null +++ b/tordam-01-overview.md @@ -0,0 +1,130 @@ +Tor DAM +======= + + +Tor Distributed Announce Mechanism (DAM) is a protocol and tooling for +mapping machines in the Tor network running this software. + +The Tor DAM network is imagined to be pseudo-distributed inside the Tor +network itself. Nodes running Tor DAM can use an existing entrypoint and +start announcing themselves to the entry point(s), or they can be their +own and let others announce to themselves. Tor DAM will store all of +these announcements in a storage backend and utilize it to expand the +knowledge of the nodes using this software. Over time the network will +keep expanding and the user will be able to see all other nodes in the +network either by querying the storage backend, or visualizing it with +some kind of software. + + +Abstract +-------- + +* Every node has a HTTP API allowing to list other nodes and announce + new ones. +* They keep propagating to all valid nodes they know. +* Announcing implies the need of knowledge of at least one or two nodes. + * It is possible to make this random enough once there are at least 6 + nodes in the network. +* A node announces itself to others by sending a JSON-formatted HTTP + POST request to one or more active node. + * Once the POST request is received, the node will validate the + request and return a secret encrypted with the requester's public + key. + * The requester will try to decrypt this secret, and return the + secret in plain text back to the node it's announcing to, along + with a cryptographic signature, so the node can confirm the + requester is in actual possession of the private key. +* Tor DAM **does not validate** if a node is malicious or not. This is a + layer that has to be established on top. Tor DAM is just the entry + point into the network. + + +Protocol +-------- + +A node announcing itself has to do a JSON-formatted HTTP POST request to +one or more active nodes with the format explained below. N.B. The +strings shown in this document might not be valid, but they represent a +correct example. + +* `type` reflects the type of the node +* `address` holds the address of the Tor hidden service +* `message` is the message that has to be signed using the private key + of this same hidden service. +* `signature` is the base64 encoded signature of the above message. +* `secret` is a string that is used for exchanging messages between the + client and server. + + +``` +{ + "type": "node", + "address": "22mobp7vrb7a4gt2.onion", + "message": "I am a DAM node!", + "signature": "BuB/Dv8E44CLzUX88K2Ab0lUNS9A0GSkHPtrFNNWZMihPMWN0ORhwMZBRnMJ8woPO3wSONBvEvaCXA2hvsVrUJTa+hnevQNyQXCRhdTVVuVXEpjyFzkMamxb6InrGqbsGGkEUqGMSr9aaQ85N02MMrM6T6JuyqSSssFg2xuO+P4=", + "secret": "" +} +``` + +Sending this as a POST request to a node will make it ask for the +public key of the given address from a "hidden service directory" +(HSDir) in the Tor network. It will retrieve the public key and try to +validate the signature that was made. Validating this, we assume that +the requester is in possession of the private key. + +Following up, the node shall generate a cryptographically secure random +string and encrypt it using the before acquired public key. It will then +be encoded using base64 and sent back to the client: + + +``` +{ + "secret": "eP07xSZWlDdK4+AL0WUkIA3OnVTc3sEgu4MUqGr43TUXaJLfAILvWxKihPxytumBmdJ4LC45LsrdDuhmUSmZZMJxxiLmB4Gf3zoWa1DmStdc147VsGpexY05jaJUZlbmG0kkTFdPmdcKNbis5xfRn8Duo1e5bOPj41lIopwiil0=" +} +``` + +The client will try to decode and decrypt this secret, and send it back +to the node to complete its part of the handshake. The POST request this +time will contain the following data: + +* `type` reflects the type of the node +* `address` holds the address of the Tor hidden service +* `message` is the decrypted and base64 encoded secret that the server + had just sent us. +* `signature` is the base64 encoded signature of the above secret. +* `secret` is a copy of `message` here. + + +``` +{ + "type": "node", + "address": "22mobp7vrb7a4gt2.onion", + "message": "ZShhYHYsRGNLOTZ6YUwwP3ZXPnxhQiR9UFVWfmk5TG56TEtLb04vMms+OTIrLlQ7aS4rflR3V041RG5Je0tnYw==", + "signature": "L1N+VEi3T3aZaYksAy1+0UMoYn7B3Gapfk0dJzOUxUtUYVhj84TgfYeDnADNYrt5UK9hN/lCTIhsM6zPO7mSjQI43l3dKvMIikqQDwNey/XaokyPI4/oKrMoGQnu8E8UmHmI1pFvwdO5EQQaKbi90qWNj93KB/NlTwqD9Ir4blY=", + "secret": "ZShhYHYsRGNLOTZ6YUwwP3ZXPnxhQiR9UFVWfmk5TG56TEtLb04vMms+OTIrLlQ7aS4rflR3V041RG5Je0tnYw==" +} +``` + +The node will verify the received plain secret against what it has +encrypted to validate. If the comparison yields no errors, we assume +that the requester is actually in possession of the private key. If the +node is not valid in our database, we will complete the handshake by +welcoming the client into the network: + + +``` +{ + "secret": "Welcome to the DAM network!" +} +``` + +Further on, the node will append useful metadata to the struct. We will +add the encoded public key, timestamps of when the client was first seen +and last seen, and a field to indicate if the node is valid. The latter +is not to be handled by Tor DAM, but rather the upper layer, which +actually has consensus handling. + +If the node is valid in another node's database, the remote node will +then propagate back all the valid nodes it knows (including itself) back +to the client in a gzipped and base64 encoded JSON struct. The client +will then handle this and update its own database accordingly. diff --git a/views/abstract.txt b/views/abstract.txt new file mode 100644 index 0000000..3a33251 --- /dev/null +++ b/views/abstract.txt @@ -0,0 +1,10 @@ +The DECODE OS is a GNU+Linux distribution for private peer-to-peer +networks of distributed computing. It is easily customisable and comes +with an intuitive web application that facilitate the task of +transforming Docker files (a popular format to prototype software +application) into OS images ready to install on embedded and cloud +infrastructure. This deliverable details the implementation details of +DECODE OS as well provides operational guidelines to deal with its +internals. As a demonstration, the web application is publicly +available on toaster.dyne.org. + diff --git a/views/index.txt b/views/index.txt new file mode 100644 index 0000000..57dbb71 --- /dev/null +++ b/views/index.txt @@ -0,0 +1,6 @@ +introduction.md +toaster-01-overview.md +tordam-01-overview.md +sdk-01-overview.md +sdk-02-blends.md +sdk-03-technical.md diff --git a/views/template.tex b/views/template.tex new file mode 100644 index 0000000..9387e0f --- /dev/null +++ b/views/template.tex @@ -0,0 +1,177 @@ +\documentclass[a4paper]{extarticle} +\usepackage{lmodern} +$if(fontsize)$ +\usepackage[$fontsize$]{extsizes} +$endif$ +\usepackage{fullpage} +\usepackage{longtable} +\usepackage{booktabs} +\usepackage{amssymb,amsmath} +\usepackage{ifxetex,ifluatex} +\usepackage{fixltx2e} % provides \textsubscript +\ifnum 0\ifxetex 1\fi\ifluatex 1\fi=0 % if pdftex + \usepackage[T1]{fontenc} + \usepackage[utf8x]{inputenc} +\else % if luatex or xelatex + \ifxetex + \usepackage{mathspec} + \else + \usepackage{fontspec} + \fi + \defaultfontfeatures{Ligatures=TeX,Scale=MatchLowercase} +\fi +% use upquote if available, for straight quotes in verbatim environments +\IfFileExists{upquote.sty}{\usepackage{upquote}}{} +% use microtype if available +\IfFileExists{microtype.sty}{% +\usepackage{microtype} +\UseMicrotypeSet[protrusion]{basicmath} % disable protrusion for tt fonts +}{} +\usepackage{hyperref} +\hypersetup{unicode=true, + pdftitle={$title$}, + pdfauthor={$author$}, + $if(keywords)$ + pdfkeywords={$for(keywords)$$keywords$$sep$; $endfor$}, + $endif$ + pdfborder={0 0 0}, + breaklinks=true} +\urlstyle{same} % don't use monospace font for urls +\usepackage{xcolor} +$if(listings)$ +\usepackage{listings} +\lstset{ + basicstyle=\ttfamily, +% numbers=left, + numberstyle=\footnotesize, + stepnumber=2, + numbersep=5pt, + backgroundcolor=\color{black!10}, + showspaces=false, + showstringspaces=false, + showtabs=false, + tabsize=2, + captionpos=b, + breaklines=true, + breakatwhitespace=true, + breakautoindent=true, + linewidth=\textwidth +} +$endif$ +\usepackage{color} +\usepackage{fancyvrb} +\newcommand{\VerbBar}{|} +\newcommand{\VERB}{\Verb[commandchars=\\\{\}]} +\DefineVerbatimEnvironment{Highlighting}{Verbatim}{commandchars=\\\{\}} +% Add ',fontsize=\small' for more characters per line +\newenvironment{Shaded}{}{} +\newcommand{\KeywordTok}[1]{\textcolor[rgb]{0.00,0.44,0.13}{\textbf{{#1}}}} +\newcommand{\DataTypeTok}[1]{\textcolor[rgb]{0.56,0.13,0.00}{{#1}}} +\newcommand{\DecValTok}[1]{\textcolor[rgb]{0.25,0.63,0.44}{{#1}}} +\newcommand{\BaseNTok}[1]{\textcolor[rgb]{0.25,0.63,0.44}{{#1}}} +\newcommand{\FloatTok}[1]{\textcolor[rgb]{0.25,0.63,0.44}{{#1}}} +\newcommand{\ConstantTok}[1]{\textcolor[rgb]{0.53,0.00,0.00}{{#1}}} +\newcommand{\CharTok}[1]{\textcolor[rgb]{0.25,0.44,0.63}{{#1}}} +\newcommand{\SpecialCharTok}[1]{\textcolor[rgb]{0.25,0.44,0.63}{{#1}}} +\newcommand{\StringTok}[1]{\textcolor[rgb]{0.25,0.44,0.63}{{#1}}} +\newcommand{\VerbatimStringTok}[1]{\textcolor[rgb]{0.25,0.44,0.63}{{#1}}} +\newcommand{\SpecialStringTok}[1]{\textcolor[rgb]{0.73,0.40,0.53}{{#1}}} +\newcommand{\ImportTok}[1]{{#1}} +\newcommand{\CommentTok}[1]{\textcolor[rgb]{0.38,0.63,0.69}{\textit{{#1}}}} +\newcommand{\DocumentationTok}[1]{\textcolor[rgb]{0.73,0.13,0.13}{\textit{{#1}}}} +\newcommand{\AnnotationTok}[1]{\textcolor[rgb]{0.38,0.63,0.69}{\textbf{\textit{{#1}}}}} +\newcommand{\CommentVarTok}[1]{\textcolor[rgb]{0.38,0.63,0.69}{\textbf{\textit{{#1}}}}} +\newcommand{\OtherTok}[1]{\textcolor[rgb]{0.00,0.44,0.13}{{#1}}} +\newcommand{\FunctionTok}[1]{\textcolor[rgb]{0.02,0.16,0.49}{{#1}}} +\newcommand{\VariableTok}[1]{\textcolor[rgb]{0.10,0.09,0.49}{{#1}}} +\newcommand{\ControlFlowTok}[1]{\textcolor[rgb]{0.00,0.44,0.13}{\textbf{{#1}}}} +\newcommand{\OperatorTok}[1]{\textcolor[rgb]{0.40,0.40,0.40}{{#1}}} +\newcommand{\BuiltInTok}[1]{{#1}} +\newcommand{\ExtensionTok}[1]{{#1}} +\newcommand{\PreprocessorTok}[1]{\textcolor[rgb]{0.74,0.48,0.00}{{#1}}} +\newcommand{\AttributeTok}[1]{\textcolor[rgb]{0.49,0.56,0.16}{{#1}}} +\newcommand{\RegionMarkerTok}[1]{{#1}} +\newcommand{\InformationTok}[1]{\textcolor[rgb]{0.38,0.63,0.69}{\textbf{\textit{{#1}}}}} +\newcommand{\WarningTok}[1]{\textcolor[rgb]{0.38,0.63,0.69}{\textbf{\textit{{#1}}}}} +\newcommand{\AlertTok}[1]{\textcolor[rgb]{1.00,0.00,0.00}{\textbf{{#1}}}} +\newcommand{\ErrorTok}[1]{\textcolor[rgb]{1.00,0.00,0.00}{\textbf{{#1}}}} +\newcommand{\NormalTok}[1]{{#1}} +\usepackage{graphicx,grffile} +\makeatletter +\def\maxwidth{\ifdim\Gin@nat@width>\linewidth\linewidth\else\Gin@nat@width\fi} +\def\maxheight{\ifdim\Gin@nat@height>\textheight\textheight\else\Gin@nat@height\fi} +\makeatother +% Scale images if necessary, so that they will not overflow the page +% margins by default, and it is still possible to overwrite the defaults +% using explicit options in \includegraphics[width, height, ...]{} +\setkeys{Gin}{width=\maxwidth,height=\maxheight,keepaspectratio} +\IfFileExists{parskip.sty}{% +\usepackage{parskip} +}{% else +\setlength{\parindent}{0pt} +\setlength{\parskip}{6pt plus 2pt minus 1pt} +} + +% previously included by writedown in options.sty +\setlength{\parindent}{1.25em} +\setlength{\parskip}{.2em} +\usepackage{etoolbox} +\AtBeginEnvironment{quote}{\parskip 1em} + +\setlength{\emergencystretch}{3em} % prevent overfull lines +\providecommand{\tightlist}{% + \setlength{\itemsep}{0pt}\setlength{\parskip}{0pt}} +\setcounter{secnumdepth}{0} +% Redefines (sub)paragraphs to behave more like sections +\ifx\paragraph\undefined\else +\let\oldparagraph\paragraph +\renewcommand{\paragraph}[1]{\oldparagraph{#1}\mbox{}} +\fi +\ifx\subparagraph\undefined\else +\let\oldsubparagraph\subparagraph +\renewcommand{\subparagraph}[1]{\oldsubparagraph{#1}\mbox{}} +\fi +% END OF CONFIG ------------------------------------------ + +% START OF CONTENT ------------------------------------------ + +\title{$title$} +$if(subtitle)$ +\providecommand{\subtitle}[1]{} +\subtitle{$subtitle$} +$endif$ +$if(author)$ +\author{$for(author)$$author$$sep$ \and $endfor$} +$endif$ +$if(institute)$ +\providecommand{\institute}[1]{} +\institute{$for(institute)$$institute$$sep$ \and $endfor$} +$endif$ +\date{$date$} +$if(logo)$ +\logo{\includegraphics{$logo$}} +$endif$ + +\begin{document} + +\maketitle + +\begin{abstract} +$abstract$ +\end{abstract} + +\providecommand{\keywords}[1]{\textbf{\textit{Keywords---}} #1} +$if(keywords)$ + \keywords{$for(keywords)$$keywords$$sep$; $endfor$} +$endif$ + +\pagebreak[4] +{ +\setcounter{tocdepth}{3} +\tableofcontents +} +\pagebreak[4] + +$body$ + +\end{document}