From cd35d77cf1636306637a49b2c909d79891b31f35 Mon Sep 17 00:00:00 2001 From: parazyd Date: Mon, 8 Jul 2019 21:30:11 +0200 Subject: [PATCH] Initial commit. --- 01-introduction.md | 18 ++++ 02-jenkins.md | 50 +++++++++++ 03-jenkins-debian-glue.md | 36 ++++++++ 04-dak.md | 4 + 05-building-packages.md | 150 ++++++++++++++++++++++++++++++++ 06-amprolla.md | 70 +++++++++++++++ 07-package-workflow.md | 79 +++++++++++++++++ config.zsh | 10 +++ pub/.keep | 0 views/abstract.txt | 1 + views/index.txt | 7 ++ views/template.tex | 177 ++++++++++++++++++++++++++++++++++++++ 12 files changed, 602 insertions(+) create mode 100644 01-introduction.md create mode 100644 02-jenkins.md create mode 100644 03-jenkins-debian-glue.md create mode 100644 04-dak.md create mode 100644 05-building-packages.md create mode 100644 06-amprolla.md create mode 100644 07-package-workflow.md create mode 100644 config.zsh create mode 100644 pub/.keep create mode 100644 views/abstract.txt create mode 100644 views/index.txt create mode 100644 views/template.tex diff --git a/01-introduction.md b/01-introduction.md new file mode 100644 index 0000000..18e472c --- /dev/null +++ b/01-introduction.md @@ -0,0 +1,18 @@ +Devuan Developers Manual +======================== + +This document contains the knowledge we've gathered while implementing +and using our continuous integration (CI) infrastructure while +developing components of Devuan and the DECODE project. + +The continuous integration infrastructure consists of + +1. The base CI system +2. Pluggable modules +3. Deployment management + +These core features of our base continuous integration ecosystem, along +with its modular pluggable components will be described in the following +sections of this document, along with any operational instructions. At +the core of our continuous integration system is the Jenkins CI +framework, hosted at . diff --git a/02-jenkins.md b/02-jenkins.md new file mode 100644 index 0000000..35fa213 --- /dev/null +++ b/02-jenkins.md @@ -0,0 +1,50 @@ +Jenkins +======= + +At the core of our CI system is the **Jenkins framework**. Jenkins +serves as the manager for all build jobs, pipelines, and artifact +collection. With its sytem of a master node and build executor nodes, we +are able to build any software or even a complete operating system +environment through cross-compilation, OS bootstrapping processes and +similar. This allows to have a build ecosystem for various CPU +architectures like x86, Arm, PowerPC, etc. + +[Jenkins](https://jenkins.io) is the leading open-source automation +server, providing hundreds of plugins to support building, deploying and +automating any project. As an extensible automation server, Jenkins can +be used as a simple CI server or turned into the continuous delivery hub +for any project. It can be easily set up and configured via its web +interface, which includes on-the-fly error checks and built-in help. +Jenkins is also able to easily distribute work across multiple machines. + +The setup is comprised of a web interface provided by Jenkins, along +with helper software that links our Gitlab instance for building Devuan +packages using a Git-based workflow. + +The build executor node ecosystem consists of various machines, +including Softiron Overdrive servers, Olimex LIME2 boards, TalosII +secure workstations, and 64-bit and 32-bit Intel/AMD computers. Jenkins +allows us to utilize all of these machines to transparently build +software for different Devuan suites (releases) and CPU architectures +without having to maintain different build system. + + +Deployment +---------- + +Reproducing the base CI ecosystem that is explained above can be done +fairly symple on **any Devuan-based** GNU/Linux distribution. To do +this, we have to run just a few simple commands: + +``` +# wget -qO- https://pkg.jenkins.io/debian-stable/jenkins.io.key | apt-key add - +# echo 'deb https://pkg.jenkins.io/debian-stable binary/' >> /etc/apt/sources.list +# apt update && apt install jenkins +``` + +Installing the `jenkins` package would result in a running Jenkins +instance, ready to produce builds. + +Further on, adding build executor nodes is as simple as configuring SSH, +installing the `jenkins-debian-glue-buildenv-devuan` package, and adding +them through Jenkins' web interface through "Add Nodes". diff --git a/03-jenkins-debian-glue.md b/03-jenkins-debian-glue.md new file mode 100644 index 0000000..8b5756e --- /dev/null +++ b/03-jenkins-debian-glue.md @@ -0,0 +1,36 @@ +jenkins-debian-glue +=================== + +As Devuan is a GNU/Linux distribution, we are usually expected to build +native _.deb_ packages of varios software and kernels. Using +[jenins-debian-glue](https://jenkins-debian-glue.org), +[dak](https://wiki.debian.org/DebianDak), and +[amprolla](https://git.devuan.org/devuan-infrastructure/amprolla3), it +is easy to produce Devuan packages and repositories that are fully +customizable for any need. + +jenkins-debian-glue is an open-source pluggable module for Jenkins that +provides Q/A test integration, policy verification, and controlled build +environments for building _deb_ packages. It holds the logic of pbuilder +and reproducible package builds. This allows us to seamlessly build +_deb_ packages that are maintained and version-controlled in Git. +Providing the entire development history and quality assurance through +code reviews and similar. + + +Deployment +---------- + +On **any Devuan-based** system, once Jenkins is installed, we can +install an additional (meta)package prepared for easy deployment of +jenkins-debian-glue called `jenkins-debian-glue-buildenv-devuan`. +Installing it, along with `default-jre-headless`, would add support to +our build nodes to build _deb_ packages using version control systems +like Subversion and Git. To install, we would simply issue: + +``` +# apt install jenkins-debian-glue-buildenv-devuan default-jre-headless +``` + +Once installed, we would be able to add and configure the node in our +Jenkins master. diff --git a/04-dak.md b/04-dak.md new file mode 100644 index 0000000..2dcae4b --- /dev/null +++ b/04-dak.md @@ -0,0 +1,4 @@ +dak +=== + +TODO diff --git a/05-building-packages.md b/05-building-packages.md new file mode 100644 index 0000000..95400a6 --- /dev/null +++ b/05-building-packages.md @@ -0,0 +1,150 @@ +Building deb packages +===================== + +Using jenkins-debian-glue, we can set up a build pipeline for any +version controlled software by creating a `debian` directory in the +repository. Its structure looks like: + +``` +debian ++-- changelog ++-- compat ++-- control ++-- gbp.conf ++-- postinst ++-- preinst ++-- rules ++-- source + +-- format +``` + +These files are equivalend to any standard Debian/Devuan package. + +Once the repository is set up, and contains a `debian` directory, we are +able to construct the _deb_ package build pipeline. It is first +necessary to create a Jenkins `-source` job, followed by a `-binaries` +job, and finishing with a `-repos` job. Further on, for easier reading, +we will consider that we are building a package called `linux-sunxi`, +which is a Linux kernel package dedicated to LIME2 Arm boards. + + +-source +------- + +The -source job is responsible to fetch the software (package) sources +and distribute them to any build executor nodes that are needed to build +the package. + +These jobs need the following configuration: + +* String parameters: + * `release` (The codename of the suite): `ascii-proposed` + * `distribution` (The upstream Devuan suite codename): `ascii` + +* Git repository URL: + * `https://git.devuan.org/devuan-packages/linux-sunxi` + * + branch(es) to be built: `suites/ascii` + +* Execution shell script: + + /usr/bin/generate-git-snapshot + +* Post-build triggers: + * Projects to build: `linux-sunxi-binaries` + * Predefined parameters: + +``` + release=${release} + distribution=${distribution} +``` + + +-binaries +--------- + +The -binaries job is responsible for compiling the source code and +creating _deb_ packages out of the compiled code. The packages are +compiled on each preconfigured build executor node, for each necessary +CPU architecture. + +These jobs need the following configuration: + +* String parameters (these are passed through by the `-source` job): + * `release` (The codename of the suite): `ascii-proposed` + * `distribution` (The upstream Devuan suite codename): `ascii` + +* Configuration matrix (for the CPU architectures) + * `architecture`: `amd64 armhf arm64` + * `label`: `amd64 armf arm64` + * + Generic combination filter: + +``` +(architecture=="amd64").implies(label=="amd64") && (architecture=="armhf").implies(label=="armhf") && (architecture=="all").implies(label=="all") && (architecture=="armel").implies(label=="armel") && (architecture=="arm64").implies(label=="arm64") +``` + +* Copy artifacts from another project: + * Name: `linux-sunxi-source` + * Artifacts to copy: `*` + +* Execution shell script: + + export BUILD_ONLY=true + /usr/bin/build-and-provide-package + +* Post-build triggers: + * Projects to build: `linux-sunxi-repos` + * Predefined parameters: + +``` + release=${release} + distribution=${distribution} + architecture=${architecture} + label=${label} +``` + +* Archive artifacts: + * `*.gz,*.bz2,*.xz,*.deb,*.udeb,*.dsc,*.changes,*.buildinfo` + + +-repos +------ + +The -repos job is responsible to take the built _deb_ packages and ship +them off to dak to create or append to an APT repository. + +These jobs need the following configuration: + +* String parameters (these are passed through by the `-binaries` job): + * `release` + * `distribution` + * `architecture` + * `label` + +* Copy artifacts from another project: + * Name: `linux-sunxi-binaries/architecture=$architecture,label=$label` + * Artifacts to copy: `*` + +* Execution shell script: + + if [ -n "$gitlabBranch" ] ; then + export codename=$gitlabBranch + fi + . /etc/jenkins/debian_glue + export KEY_ID + for i in architecture* ; do + #cp $i/* . + cd $i + debsign --no-re-sign -k$KEY_ID *.changes || true + debsign --no-re-sign -k$KEY_ID *.dsc || true + cd .. + done + #find . -type d -name 'architecture*' | xargs rm -r + #debsign --no-re-sign -k$KEY_ID *.changes + #debsign --no-re-sign -k$KEY_ID *.dsc + ssh dak@repo.devuan.org mkdir /home/dak/jenkins/$BUILD_TAG + scp -r * dak@repo.devuan.org:/home/dak/jenkins/$BUILD_TAG + ssh dak@repo.devuan.org dak_add_pkgs -s $codename -c main $BUILD_TAG + + +Once this pipeline completes, the repository is ready to be used, but it +is also possible to utilize **amprolla** for additional customization. diff --git a/06-amprolla.md b/06-amprolla.md new file mode 100644 index 0000000..78098cd --- /dev/null +++ b/06-amprolla.md @@ -0,0 +1,70 @@ +amprolla +======== + +amprolla is a tool that will merge a number of different APT-based +repositories into one, while giving control over (not) including given +packages, architectures, or any specific package metadata. Upon +completing the merge, amprolla will generate and optionally create GnuPG +signatures of the according `Release` files. + +amprolla is able to run on any system supporting Python3, rsync, and +GnuPG. + +To configure amprolla, a default configuration file is provided in +`lib/config.def.py`. Copy the file to `lib/config.py` and edit it to +what is needed. The configuration file contains all the information +needed to properly merge the required repositories. The default +configuration also works, as long as a valid GPG fingerprint is provided +to sign the Release files. + +The `*dir` variables in the configuration file are the directories where +the files that are being merged are kept, and the merges itself are +done. They can be either absolute or relative paths to the root amprolla +directory. The preferred way is to actually have absolute paths. + +`banpkgs` is a set of package names that amprolla will refuse to merge +if they are found either in the dependencies of a package or if they are +the package itself. + +`repo_order` is a list that holds what is ordered in the priority the +packages are preferred. The preference is ordered first to last. The +dict `repos` holds their required information. + +To avoid unnecessary duplication, further documentation is available in +amprolla's source code. + + +Usage +----- + +After setup, it is needed to perform an initial full download and full +merge. First run `amprolla_init.py`, which is going to download the +necessary directory structures (as defined through the config file) we +will merge afterwards. When the download is done, it is time to perform +the full initial merge of these repositories. This will provide us with +a complete merged repository and we will then be able to easily perform +incremental updates. + +After the initial merge has been performed, it is advisable to run a +script called `populate_aliases.sh` found in the `contrib` directory. +Make sure it's properly edited and configured. + +To merge `Contents` files, run `amprolla_merge_contents.py`. This module +does not do incremental updates and should not be ran often due to its +heavy IO/RAM requirements. + +Incremental updates are performed through `amprolla_update.py`, however, +for more stable performance and uptime, the incremental updating is +being orchestrated by a shell script called `orchestrate.sh`. This shell +script holds the logic to have near-atomic switching of repositories to +minimize repo downtime during performed merges. Not doing this could +result in users downloading corrupted repository files if they are +requested during an ongoing merge. + +In a terminal session, simply execute the `orchestrate.sh` script and it +will start looping and doing incremental updates in a specific +timeframe. If prefered, this script can be used with cron as well. + +To actually serve the merged directory over HTTP(S), a basic nginx +configuration is provided as `contrib/nginx.conf`, and a lighttpd conf +is provided in `contrib/lighttpd.conf`. diff --git a/07-package-workflow.md b/07-package-workflow.md new file mode 100644 index 0000000..3f1991b --- /dev/null +++ b/07-package-workflow.md @@ -0,0 +1,79 @@ +Package workflow +================ + +When it comes to creating, maintaining, and building packages in Devuan, +there is a standardized workflow that has been used so far. All of +Devuan's official packages reside in the `devuan-packages` namespace on +our [GitLab](https://git.devuan.org) instance. Once a package is being +maintained under this namespace, it can be considered usable and ready +to build. + +A Devuan package's `debian` directory should be maintained the same way +like any other Debian package, with the possible exception of `gbp.conf`. +In `gbp.conf` we have to set up the upstream tag for the tag we are +planning to build. An example gbp.conf file looks like: + +``` +[DEFAULT] +upstram-tag = %(version)s +``` + +This helps jenkins-debian-glue, and more specifically - pbuilder - to +figure out what to do with the git repository and how to build the +package. + +In Devuan, we maintain many different suites, like `ascii-proposed`, +`ascii-updates`, `ascii-proposed`, etc. All of these are then supposed +to be maintained in different branches, that are named like this, with +the additional prefix of `suites/`, so an example branch would be: +`suites/ascii`. The package in this git repository under this branch +would then be build and it would end up in the ascii suite. + + +An example workflow +------------------- + +In shell commands, introducing a package into Devuan would look like the +following: + +``` +$ git clone myproject.git +$ cd myproject + +## Then add a debian directory and fill up the needed information + +$ git checkout -b suites/ascii + +$ git tag 0.1 + +## Set up remote for pushing to git.devuan.org + +$ git push --tags +``` + +This would get a package ready for building. + + +Following up, we need to get it on the Jenkins CI... + +### Pushing to Jenkins + +Once we have our package ready in the `devuan-packages` namespace, we +push it to Jenkins by opening a Gitlab issue: + +* Title: `buildadd` +* Assignee: `@autobuild` +* Labels: `any` + +(The labels here correspond to CPU architectures). + +After opening the issue, in a matter of minutes, a bot will scan the +issue, create corresponding jobs on Jenkins, and close your issue, +commenting its progress and end result. + +To build your package, open another issue: + +* Title: `build: +* Assignee: `@autobuild` + +And again, in a matter of minutes, you package should start building. diff --git a/config.zsh b/config.zsh new file mode 100644 index 0000000..96c6180 --- /dev/null +++ b/config.zsh @@ -0,0 +1,10 @@ +WRITEDOWN_TITLE="Devuan Developers Manual" +WRITEDOWN_AUTHOR="Ivan (parazyd) Jelincic" +WRITEDOWN_AFFILIATION="Dyne.org Foundation" +WRITEDOWN_DATE="8 Jul 2019" +WRITEDOWN_TAGS="[continuous, software, integration, os, linux, distro]" + +WRITEDOWN_NRSEC=yes +WRITEDOWN_TOC=yes +WRITEDOWN_CITSTYLE=harvard-kings-college-london +WRITEDOWN_FONTSIZE=14pt diff --git a/pub/.keep b/pub/.keep new file mode 100644 index 0000000..e69de29 diff --git a/views/abstract.txt b/views/abstract.txt new file mode 100644 index 0000000..001c614 --- /dev/null +++ b/views/abstract.txt @@ -0,0 +1 @@ +Devuan packaging workflow and other useful information. diff --git a/views/index.txt b/views/index.txt new file mode 100644 index 0000000..ff37f1f --- /dev/null +++ b/views/index.txt @@ -0,0 +1,7 @@ +01-introduction.md +02-jenkins.md +03-jenkins-debian-glue.md +04-dak.md +05-building-packages.md +06-amprolla.md +07-package-workflow.md diff --git a/views/template.tex b/views/template.tex new file mode 100644 index 0000000..9387e0f --- /dev/null +++ b/views/template.tex @@ -0,0 +1,177 @@ +\documentclass[a4paper]{extarticle} +\usepackage{lmodern} +$if(fontsize)$ +\usepackage[$fontsize$]{extsizes} +$endif$ +\usepackage{fullpage} +\usepackage{longtable} +\usepackage{booktabs} +\usepackage{amssymb,amsmath} +\usepackage{ifxetex,ifluatex} +\usepackage{fixltx2e} % provides \textsubscript +\ifnum 0\ifxetex 1\fi\ifluatex 1\fi=0 % if pdftex + \usepackage[T1]{fontenc} + \usepackage[utf8x]{inputenc} +\else % if luatex or xelatex + \ifxetex + \usepackage{mathspec} + \else + \usepackage{fontspec} + \fi + \defaultfontfeatures{Ligatures=TeX,Scale=MatchLowercase} +\fi +% use upquote if available, for straight quotes in verbatim environments +\IfFileExists{upquote.sty}{\usepackage{upquote}}{} +% use microtype if available +\IfFileExists{microtype.sty}{% +\usepackage{microtype} +\UseMicrotypeSet[protrusion]{basicmath} % disable protrusion for tt fonts +}{} +\usepackage{hyperref} +\hypersetup{unicode=true, + pdftitle={$title$}, + pdfauthor={$author$}, + $if(keywords)$ + pdfkeywords={$for(keywords)$$keywords$$sep$; $endfor$}, + $endif$ + pdfborder={0 0 0}, + breaklinks=true} +\urlstyle{same} % don't use monospace font for urls +\usepackage{xcolor} +$if(listings)$ +\usepackage{listings} +\lstset{ + basicstyle=\ttfamily, +% numbers=left, + numberstyle=\footnotesize, + stepnumber=2, + numbersep=5pt, + backgroundcolor=\color{black!10}, + showspaces=false, + showstringspaces=false, + showtabs=false, + tabsize=2, + captionpos=b, + breaklines=true, + breakatwhitespace=true, + breakautoindent=true, + linewidth=\textwidth +} +$endif$ +\usepackage{color} +\usepackage{fancyvrb} +\newcommand{\VerbBar}{|} +\newcommand{\VERB}{\Verb[commandchars=\\\{\}]} +\DefineVerbatimEnvironment{Highlighting}{Verbatim}{commandchars=\\\{\}} +% Add ',fontsize=\small' for more characters per line +\newenvironment{Shaded}{}{} +\newcommand{\KeywordTok}[1]{\textcolor[rgb]{0.00,0.44,0.13}{\textbf{{#1}}}} +\newcommand{\DataTypeTok}[1]{\textcolor[rgb]{0.56,0.13,0.00}{{#1}}} +\newcommand{\DecValTok}[1]{\textcolor[rgb]{0.25,0.63,0.44}{{#1}}} +\newcommand{\BaseNTok}[1]{\textcolor[rgb]{0.25,0.63,0.44}{{#1}}} +\newcommand{\FloatTok}[1]{\textcolor[rgb]{0.25,0.63,0.44}{{#1}}} +\newcommand{\ConstantTok}[1]{\textcolor[rgb]{0.53,0.00,0.00}{{#1}}} +\newcommand{\CharTok}[1]{\textcolor[rgb]{0.25,0.44,0.63}{{#1}}} +\newcommand{\SpecialCharTok}[1]{\textcolor[rgb]{0.25,0.44,0.63}{{#1}}} +\newcommand{\StringTok}[1]{\textcolor[rgb]{0.25,0.44,0.63}{{#1}}} +\newcommand{\VerbatimStringTok}[1]{\textcolor[rgb]{0.25,0.44,0.63}{{#1}}} +\newcommand{\SpecialStringTok}[1]{\textcolor[rgb]{0.73,0.40,0.53}{{#1}}} +\newcommand{\ImportTok}[1]{{#1}} +\newcommand{\CommentTok}[1]{\textcolor[rgb]{0.38,0.63,0.69}{\textit{{#1}}}} +\newcommand{\DocumentationTok}[1]{\textcolor[rgb]{0.73,0.13,0.13}{\textit{{#1}}}} +\newcommand{\AnnotationTok}[1]{\textcolor[rgb]{0.38,0.63,0.69}{\textbf{\textit{{#1}}}}} +\newcommand{\CommentVarTok}[1]{\textcolor[rgb]{0.38,0.63,0.69}{\textbf{\textit{{#1}}}}} +\newcommand{\OtherTok}[1]{\textcolor[rgb]{0.00,0.44,0.13}{{#1}}} +\newcommand{\FunctionTok}[1]{\textcolor[rgb]{0.02,0.16,0.49}{{#1}}} +\newcommand{\VariableTok}[1]{\textcolor[rgb]{0.10,0.09,0.49}{{#1}}} +\newcommand{\ControlFlowTok}[1]{\textcolor[rgb]{0.00,0.44,0.13}{\textbf{{#1}}}} +\newcommand{\OperatorTok}[1]{\textcolor[rgb]{0.40,0.40,0.40}{{#1}}} +\newcommand{\BuiltInTok}[1]{{#1}} +\newcommand{\ExtensionTok}[1]{{#1}} +\newcommand{\PreprocessorTok}[1]{\textcolor[rgb]{0.74,0.48,0.00}{{#1}}} +\newcommand{\AttributeTok}[1]{\textcolor[rgb]{0.49,0.56,0.16}{{#1}}} +\newcommand{\RegionMarkerTok}[1]{{#1}} +\newcommand{\InformationTok}[1]{\textcolor[rgb]{0.38,0.63,0.69}{\textbf{\textit{{#1}}}}} +\newcommand{\WarningTok}[1]{\textcolor[rgb]{0.38,0.63,0.69}{\textbf{\textit{{#1}}}}} +\newcommand{\AlertTok}[1]{\textcolor[rgb]{1.00,0.00,0.00}{\textbf{{#1}}}} +\newcommand{\ErrorTok}[1]{\textcolor[rgb]{1.00,0.00,0.00}{\textbf{{#1}}}} +\newcommand{\NormalTok}[1]{{#1}} +\usepackage{graphicx,grffile} +\makeatletter +\def\maxwidth{\ifdim\Gin@nat@width>\linewidth\linewidth\else\Gin@nat@width\fi} +\def\maxheight{\ifdim\Gin@nat@height>\textheight\textheight\else\Gin@nat@height\fi} +\makeatother +% Scale images if necessary, so that they will not overflow the page +% margins by default, and it is still possible to overwrite the defaults +% using explicit options in \includegraphics[width, height, ...]{} +\setkeys{Gin}{width=\maxwidth,height=\maxheight,keepaspectratio} +\IfFileExists{parskip.sty}{% +\usepackage{parskip} +}{% else +\setlength{\parindent}{0pt} +\setlength{\parskip}{6pt plus 2pt minus 1pt} +} + +% previously included by writedown in options.sty +\setlength{\parindent}{1.25em} +\setlength{\parskip}{.2em} +\usepackage{etoolbox} +\AtBeginEnvironment{quote}{\parskip 1em} + +\setlength{\emergencystretch}{3em} % prevent overfull lines +\providecommand{\tightlist}{% + \setlength{\itemsep}{0pt}\setlength{\parskip}{0pt}} +\setcounter{secnumdepth}{0} +% Redefines (sub)paragraphs to behave more like sections +\ifx\paragraph\undefined\else +\let\oldparagraph\paragraph +\renewcommand{\paragraph}[1]{\oldparagraph{#1}\mbox{}} +\fi +\ifx\subparagraph\undefined\else +\let\oldsubparagraph\subparagraph +\renewcommand{\subparagraph}[1]{\oldsubparagraph{#1}\mbox{}} +\fi +% END OF CONFIG ------------------------------------------ + +% START OF CONTENT ------------------------------------------ + +\title{$title$} +$if(subtitle)$ +\providecommand{\subtitle}[1]{} +\subtitle{$subtitle$} +$endif$ +$if(author)$ +\author{$for(author)$$author$$sep$ \and $endfor$} +$endif$ +$if(institute)$ +\providecommand{\institute}[1]{} +\institute{$for(institute)$$institute$$sep$ \and $endfor$} +$endif$ +\date{$date$} +$if(logo)$ +\logo{\includegraphics{$logo$}} +$endif$ + +\begin{document} + +\maketitle + +\begin{abstract} +$abstract$ +\end{abstract} + +\providecommand{\keywords}[1]{\textbf{\textit{Keywords---}} #1} +$if(keywords)$ + \keywords{$for(keywords)$$keywords$$sep$; $endfor$} +$endif$ + +\pagebreak[4] +{ +\setcounter{tocdepth}{3} +\tableofcontents +} +\pagebreak[4] + +$body$ + +\end{document}