cconn a transparent connection wrapper

cconn used the inventory system to dynamically connect to FreeBSD jail via
the host system and/or using sudo to get root access

This is more of a prove of concept then a finish idea.

.gitattributes

@@ -0,0 +1,8 @@
+.gitignore export-ignore
+.gitattributes export-ignore
+.gitkeep export-ignore
+docs/speeches export-ignore
+docs/video export-ignore
+docs/src/man7 export-ignore
+bin/cdist-build-helper export-ignore
+README-maintainers export-ignore

.gitignore

@@ -1,5 +1,22 @@
 # -vim
-.*.swp
+# Swap
+[._]*.s[a-v][a-z]
+[._]*.sw[a-p]
+[._]s[a-rt-v][a-z]
+[._]ss[a-gi-z]
+[._]sw[a-p]
+
+# Session
+Session.vim
+
+# Temporary
+.netrwhist
+*~
+*.tmp
+# Auto-generated tag files
+tags
+# Persistent undo
+[._]*.un~
 
 # Ignore generated manpages
 docs/src/.marker
@@ -7,6 +24,8 @@ docs/src/man1/*.1
 docs/src/man7/*.7
 docs/src/man7/cdist-type__*.rst
 docs/src/cdist-reference.rst
+docs/src/cdist-types.rst
+docs/src/cdist.cfg.skeleton
 
 # Ignore cdist cache for version control
 /cache/
@@ -17,15 +36,17 @@ cdist/inventory/
 # Python: cache, distutils, distribution in general
 __pycache__/
 *.pyc
-MANIFEST
+/MANIFEST
 dist/
 cdist/version.py
+cdist.egg-info/
 
 # sphinx build dirs, cache
 _build/
 docs/dist
 
 # Ignore temp files used for signing
+cdist-*.tar
 cdist-*.tar.gz
 cdist-*.tar.gz.asc
 

.gitlab-ci.yml

@@ -0,0 +1,23 @@
+---
+image: code.ungleich.ch:5050/ungleich-public/cdist/cdist-ci:latest
+
+stages:
+    - test
+
+before_script:
+    - ./bin/cdist-build-helper version
+
+shellcheck:
+    stage: test
+    script:
+        - ./bin/cdist-build-helper shellcheck
+
+pycodestyle:
+    stage: test
+    script:
+        - ./bin/cdist-build-helper pycodestyle
+
+unit_tests:
+    stage: test
+    script:
+        - ./bin/cdist-build-helper test

LICENSE

@@ -0,0 +1,674 @@
+                    GNU GENERAL PUBLIC LICENSE
+                       Version 3, 29 June 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+                            Preamble
+
+  The GNU General Public License is a free, copyleft license for
+software and other kinds of works.
+
+  The licenses for most software and other practical works are designed
+to take away your freedom to share and change the works.  By contrast,
+the GNU General Public License is intended to guarantee your freedom to
+share and change all versions of a program--to make sure it remains free
+software for all its users.  We, the Free Software Foundation, use the
+GNU General Public License for most of our software; it applies also to
+any other work released this way by its authors.  You can apply it to
+your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+them if you wish), that you receive source code or can get it if you
+want it, that you can change the software or use pieces of it in new
+free programs, and that you know you can do these things.
+
+  To protect your rights, we need to prevent others from denying you
+these rights or asking you to surrender the rights.  Therefore, you have
+certain responsibilities if you distribute copies of the software, or if
+you modify it: responsibilities to respect the freedom of others.
+
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must pass on to the recipients the same
+freedoms that you received.  You must make sure that they, too, receive
+or can get the source code.  And you must show them these terms so they
+know their rights.
+
+  Developers that use the GNU GPL protect your rights with two steps:
+(1) assert copyright on the software, and (2) offer you this License
+giving you legal permission to copy, distribute and/or modify it.
+
+  For the developers' and authors' protection, the GPL clearly explains
+that there is no warranty for this free software.  For both users' and
+authors' sake, the GPL requires that modified versions be marked as
+changed, so that their problems will not be attributed erroneously to
+authors of previous versions.
+
+  Some devices are designed to deny users access to install or run
+modified versions of the software inside them, although the manufacturer
+can do so.  This is fundamentally incompatible with the aim of
+protecting users' freedom to change the software.  The systematic
+pattern of such abuse occurs in the area of products for individuals to
+use, which is precisely where it is most unacceptable.  Therefore, we
+have designed this version of the GPL to prohibit the practice for those
+products.  If such problems arise substantially in other domains, we
+stand ready to extend this provision to those domains in future versions
+of the GPL, as needed to protect the freedom of users.
+
+  Finally, every program is threatened constantly by software patents.
+States should not allow patents to restrict development and use of
+software on general-purpose computers, but in those that do, we wish to
+avoid the special danger that patents applied to a free program could
+make it effectively proprietary.  To prevent this, the GPL assures that
+patents cannot be used to render the program non-free.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+                       TERMS AND CONDITIONS
+
+  0. Definitions.
+
+  "This License" refers to version 3 of the GNU General Public License.
+
+  "Copyright" also means copyright-like laws that apply to other kinds of
+works, such as semiconductor masks.
+
+  "The Program" refers to any copyrightable work licensed under this
+License.  Each licensee is addressed as "you".  "Licensees" and
+"recipients" may be individuals or organizations.
+
+  To "modify" a work means to copy from or adapt all or part of the work
+in a fashion requiring copyright permission, other than the making of an
+exact copy.  The resulting work is called a "modified version" of the
+earlier work or a work "based on" the earlier work.
+
+  A "covered work" means either the unmodified Program or a work based
+on the Program.
+
+  To "propagate" a work means to do anything with it that, without
+permission, would make you directly or secondarily liable for
+infringement under applicable copyright law, except executing it on a
+computer or modifying a private copy.  Propagation includes copying,
+distribution (with or without modification), making available to the
+public, and in some countries other activities as well.
+
+  To "convey" a work means any kind of propagation that enables other
+parties to make or receive copies.  Mere interaction with a user through
+a computer network, with no transfer of a copy, is not conveying.
+
+  An interactive user interface displays "Appropriate Legal Notices"
+to the extent that it includes a convenient and prominently visible
+feature that (1) displays an appropriate copyright notice, and (2)
+tells the user that there is no warranty for the work (except to the
+extent that warranties are provided), that licensees may convey the
+work under this License, and how to view a copy of this License.  If
+the interface presents a list of user commands or options, such as a
+menu, a prominent item in the list meets this criterion.
+
+  1. Source Code.
+
+  The "source code" for a work means the preferred form of the work
+for making modifications to it.  "Object code" means any non-source
+form of a work.
+
+  A "Standard Interface" means an interface that either is an official
+standard defined by a recognized standards body, or, in the case of
+interfaces specified for a particular programming language, one that
+is widely used among developers working in that language.
+
+  The "System Libraries" of an executable work include anything, other
+than the work as a whole, that (a) is included in the normal form of
+packaging a Major Component, but which is not part of that Major
+Component, and (b) serves only to enable use of the work with that
+Major Component, or to implement a Standard Interface for which an
+implementation is available to the public in source code form.  A
+"Major Component", in this context, means a major essential component
+(kernel, window system, and so on) of the specific operating system
+(if any) on which the executable work runs, or a compiler used to
+produce the work, or an object code interpreter used to run it.
+
+  The "Corresponding Source" for a work in object code form means all
+the source code needed to generate, install, and (for an executable
+work) run the object code and to modify the work, including scripts to
+control those activities.  However, it does not include the work's
+System Libraries, or general-purpose tools or generally available free
+programs which are used unmodified in performing those activities but
+which are not part of the work.  For example, Corresponding Source
+includes interface definition files associated with source files for
+the work, and the source code for shared libraries and dynamically
+linked subprograms that the work is specifically designed to require,
+such as by intimate data communication or control flow between those
+subprograms and other parts of the work.
+
+  The Corresponding Source need not include anything that users
+can regenerate automatically from other parts of the Corresponding
+Source.
+
+  The Corresponding Source for a work in source code form is that
+same work.
+
+  2. Basic Permissions.
+
+  All rights granted under this License are granted for the term of
+copyright on the Program, and are irrevocable provided the stated
+conditions are met.  This License explicitly affirms your unlimited
+permission to run the unmodified Program.  The output from running a
+covered work is covered by this License only if the output, given its
+content, constitutes a covered work.  This License acknowledges your
+rights of fair use or other equivalent, as provided by copyright law.
+
+  You may make, run and propagate covered works that you do not
+convey, without conditions so long as your license otherwise remains
+in force.  You may convey covered works to others for the sole purpose
+of having them make modifications exclusively for you, or provide you
+with facilities for running those works, provided that you comply with
+the terms of this License in conveying all material for which you do
+not control copyright.  Those thus making or running the covered works
+for you must do so exclusively on your behalf, under your direction
+and control, on terms that prohibit them from making any copies of
+your copyrighted material outside their relationship with you.
+
+  Conveying under any other circumstances is permitted solely under
+the conditions stated below.  Sublicensing is not allowed; section 10
+makes it unnecessary.
+
+  3. Protecting Users' Legal Rights From Anti-Circumvention Law.
+
+  No covered work shall be deemed part of an effective technological
+measure under any applicable law fulfilling obligations under article
+11 of the WIPO copyright treaty adopted on 20 December 1996, or
+similar laws prohibiting or restricting circumvention of such
+measures.
+
+  When you convey a covered work, you waive any legal power to forbid
+circumvention of technological measures to the extent such circumvention
+is effected by exercising rights under this License with respect to
+the covered work, and you disclaim any intention to limit operation or
+modification of the work as a means of enforcing, against the work's
+users, your or third parties' legal rights to forbid circumvention of
+technological measures.
+
+  4. Conveying Verbatim Copies.
+
+  You may convey verbatim copies of the Program's source code as you
+receive it, in any medium, provided that you conspicuously and
+appropriately publish on each copy an appropriate copyright notice;
+keep intact all notices stating that this License and any
+non-permissive terms added in accord with section 7 apply to the code;
+keep intact all notices of the absence of any warranty; and give all
+recipients a copy of this License along with the Program.
+
+  You may charge any price or no price for each copy that you convey,
+and you may offer support or warranty protection for a fee.
+
+  5. Conveying Modified Source Versions.
+
+  You may convey a work based on the Program, or the modifications to
+produce it from the Program, in the form of source code under the
+terms of section 4, provided that you also meet all of these conditions:
+
+    a) The work must carry prominent notices stating that you modified
+    it, and giving a relevant date.
+
+    b) The work must carry prominent notices stating that it is
+    released under this License and any conditions added under section
+    7.  This requirement modifies the requirement in section 4 to
+    "keep intact all notices".
+
+    c) You must license the entire work, as a whole, under this
+    License to anyone who comes into possession of a copy.  This
+    License will therefore apply, along with any applicable section 7
+    additional terms, to the whole of the work, and all its parts,
+    regardless of how they are packaged.  This License gives no
+    permission to license the work in any other way, but it does not
+    invalidate such permission if you have separately received it.
+
+    d) If the work has interactive user interfaces, each must display
+    Appropriate Legal Notices; however, if the Program has interactive
+    interfaces that do not display Appropriate Legal Notices, your
+    work need not make them do so.
+
+  A compilation of a covered work with other separate and independent
+works, which are not by their nature extensions of the covered work,
+and which are not combined with it such as to form a larger program,
+in or on a volume of a storage or distribution medium, is called an
+"aggregate" if the compilation and its resulting copyright are not
+used to limit the access or legal rights of the compilation's users
+beyond what the individual works permit.  Inclusion of a covered work
+in an aggregate does not cause this License to apply to the other
+parts of the aggregate.
+
+  6. Conveying Non-Source Forms.
+
+  You may convey a covered work in object code form under the terms
+of sections 4 and 5, provided that you also convey the
+machine-readable Corresponding Source under the terms of this License,
+in one of these ways:
+
+    a) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by the
+    Corresponding Source fixed on a durable physical medium
+    customarily used for software interchange.
+
+    b) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by a
+    written offer, valid for at least three years and valid for as
+    long as you offer spare parts or customer support for that product
+    model, to give anyone who possesses the object code either (1) a
+    copy of the Corresponding Source for all the software in the
+    product that is covered by this License, on a durable physical
+    medium customarily used for software interchange, for a price no
+    more than your reasonable cost of physically performing this
+    conveying of source, or (2) access to copy the
+    Corresponding Source from a network server at no charge.
+
+    c) Convey individual copies of the object code with a copy of the
+    written offer to provide the Corresponding Source.  This
+    alternative is allowed only occasionally and noncommercially, and
+    only if you received the object code with such an offer, in accord
+    with subsection 6b.
+
+    d) Convey the object code by offering access from a designated
+    place (gratis or for a charge), and offer equivalent access to the
+    Corresponding Source in the same way through the same place at no
+    further charge.  You need not require recipients to copy the
+    Corresponding Source along with the object code.  If the place to
+    copy the object code is a network server, the Corresponding Source
+    may be on a different server (operated by you or a third party)
+    that supports equivalent copying facilities, provided you maintain
+    clear directions next to the object code saying where to find the
+    Corresponding Source.  Regardless of what server hosts the
+    Corresponding Source, you remain obligated to ensure that it is
+    available for as long as needed to satisfy these requirements.
+
+    e) Convey the object code using peer-to-peer transmission, provided
+    you inform other peers where the object code and Corresponding
+    Source of the work are being offered to the general public at no
+    charge under subsection 6d.
+
+  A separable portion of the object code, whose source code is excluded
+from the Corresponding Source as a System Library, need not be
+included in conveying the object code work.
+
+  A "User Product" is either (1) a "consumer product", which means any
+tangible personal property which is normally used for personal, family,
+or household purposes, or (2) anything designed or sold for incorporation
+into a dwelling.  In determining whether a product is a consumer product,
+doubtful cases shall be resolved in favor of coverage.  For a particular
+product received by a particular user, "normally used" refers to a
+typical or common use of that class of product, regardless of the status
+of the particular user or of the way in which the particular user
+actually uses, or expects or is expected to use, the product.  A product
+is a consumer product regardless of whether the product has substantial
+commercial, industrial or non-consumer uses, unless such uses represent
+the only significant mode of use of the product.
+
+  "Installation Information" for a User Product means any methods,
+procedures, authorization keys, or other information required to install
+and execute modified versions of a covered work in that User Product from
+a modified version of its Corresponding Source.  The information must
+suffice to ensure that the continued functioning of the modified object
+code is in no case prevented or interfered with solely because
+modification has been made.
+
+  If you convey an object code work under this section in, or with, or
+specifically for use in, a User Product, and the conveying occurs as
+part of a transaction in which the right of possession and use of the
+User Product is transferred to the recipient in perpetuity or for a
+fixed term (regardless of how the transaction is characterized), the
+Corresponding Source conveyed under this section must be accompanied
+by the Installation Information.  But this requirement does not apply
+if neither you nor any third party retains the ability to install
+modified object code on the User Product (for example, the work has
+been installed in ROM).
+
+  The requirement to provide Installation Information does not include a
+requirement to continue to provide support service, warranty, or updates
+for a work that has been modified or installed by the recipient, or for
+the User Product in which it has been modified or installed.  Access to a
+network may be denied when the modification itself materially and
+adversely affects the operation of the network or violates the rules and
+protocols for communication across the network.
+
+  Corresponding Source conveyed, and Installation Information provided,
+in accord with this section must be in a format that is publicly
+documented (and with an implementation available to the public in
+source code form), and must require no special password or key for
+unpacking, reading or copying.
+
+  7. Additional Terms.
+
+  "Additional permissions" are terms that supplement the terms of this
+License by making exceptions from one or more of its conditions.
+Additional permissions that are applicable to the entire Program shall
+be treated as though they were included in this License, to the extent
+that they are valid under applicable law.  If additional permissions
+apply only to part of the Program, that part may be used separately
+under those permissions, but the entire Program remains governed by
+this License without regard to the additional permissions.
+
+  When you convey a copy of a covered work, you may at your option
+remove any additional permissions from that copy, or from any part of
+it.  (Additional permissions may be written to require their own
+removal in certain cases when you modify the work.)  You may place
+additional permissions on material, added by you to a covered work,
+for which you have or can give appropriate copyright permission.
+
+  Notwithstanding any other provision of this License, for material you
+add to a covered work, you may (if authorized by the copyright holders of
+that material) supplement the terms of this License with terms:
+
+    a) Disclaiming warranty or limiting liability differently from the
+    terms of sections 15 and 16 of this License; or
+
+    b) Requiring preservation of specified reasonable legal notices or
+    author attributions in that material or in the Appropriate Legal
+    Notices displayed by works containing it; or
+
+    c) Prohibiting misrepresentation of the origin of that material, or
+    requiring that modified versions of such material be marked in
+    reasonable ways as different from the original version; or
+
+    d) Limiting the use for publicity purposes of names of licensors or
+    authors of the material; or
+
+    e) Declining to grant rights under trademark law for use of some
+    trade names, trademarks, or service marks; or
+
+    f) Requiring indemnification of licensors and authors of that
+    material by anyone who conveys the material (or modified versions of
+    it) with contractual assumptions of liability to the recipient, for
+    any liability that these contractual assumptions directly impose on
+    those licensors and authors.
+
+  All other non-permissive additional terms are considered "further
+restrictions" within the meaning of section 10.  If the Program as you
+received it, or any part of it, contains a notice stating that it is
+governed by this License along with a term that is a further
+restriction, you may remove that term.  If a license document contains
+a further restriction but permits relicensing or conveying under this
+License, you may add to a covered work material governed by the terms
+of that license document, provided that the further restriction does
+not survive such relicensing or conveying.
+
+  If you add terms to a covered work in accord with this section, you
+must place, in the relevant source files, a statement of the
+additional terms that apply to those files, or a notice indicating
+where to find the applicable terms.
+
+  Additional terms, permissive or non-permissive, may be stated in the
+form of a separately written license, or stated as exceptions;
+the above requirements apply either way.
+
+  8. Termination.
+
+  You may not propagate or modify a covered work except as expressly
+provided under this License.  Any attempt otherwise to propagate or
+modify it is void, and will automatically terminate your rights under
+this License (including any patent licenses granted under the third
+paragraph of section 11).
+
+  However, if you cease all violation of this License, then your
+license from a particular copyright holder is reinstated (a)
+provisionally, unless and until the copyright holder explicitly and
+finally terminates your license, and (b) permanently, if the copyright
+holder fails to notify you of the violation by some reasonable means
+prior to 60 days after the cessation.
+
+  Moreover, your license from a particular copyright holder is
+reinstated permanently if the copyright holder notifies you of the
+violation by some reasonable means, this is the first time you have
+received notice of violation of this License (for any work) from that
+copyright holder, and you cure the violation prior to 30 days after
+your receipt of the notice.
+
+  Termination of your rights under this section does not terminate the
+licenses of parties who have received copies or rights from you under
+this License.  If your rights have been terminated and not permanently
+reinstated, you do not qualify to receive new licenses for the same
+material under section 10.
+
+  9. Acceptance Not Required for Having Copies.
+
+  You are not required to accept this License in order to receive or
+run a copy of the Program.  Ancillary propagation of a covered work
+occurring solely as a consequence of using peer-to-peer transmission
+to receive a copy likewise does not require acceptance.  However,
+nothing other than this License grants you permission to propagate or
+modify any covered work.  These actions infringe copyright if you do
+not accept this License.  Therefore, by modifying or propagating a
+covered work, you indicate your acceptance of this License to do so.
+
+  10. Automatic Licensing of Downstream Recipients.
+
+  Each time you convey a covered work, the recipient automatically
+receives a license from the original licensors, to run, modify and
+propagate that work, subject to this License.  You are not responsible
+for enforcing compliance by third parties with this License.
+
+  An "entity transaction" is a transaction transferring control of an
+organization, or substantially all assets of one, or subdividing an
+organization, or merging organizations.  If propagation of a covered
+work results from an entity transaction, each party to that
+transaction who receives a copy of the work also receives whatever
+licenses to the work the party's predecessor in interest had or could
+give under the previous paragraph, plus a right to possession of the
+Corresponding Source of the work from the predecessor in interest, if
+the predecessor has it or can get it with reasonable efforts.
+
+  You may not impose any further restrictions on the exercise of the
+rights granted or affirmed under this License.  For example, you may
+not impose a license fee, royalty, or other charge for exercise of
+rights granted under this License, and you may not initiate litigation
+(including a cross-claim or counterclaim in a lawsuit) alleging that
+any patent claim is infringed by making, using, selling, offering for
+sale, or importing the Program or any portion of it.
+
+  11. Patents.
+
+  A "contributor" is a copyright holder who authorizes use under this
+License of the Program or a work on which the Program is based.  The
+work thus licensed is called the contributor's "contributor version".
+
+  A contributor's "essential patent claims" are all patent claims
+owned or controlled by the contributor, whether already acquired or
+hereafter acquired, that would be infringed by some manner, permitted
+by this License, of making, using, or selling its contributor version,
+but do not include claims that would be infringed only as a
+consequence of further modification of the contributor version.  For
+purposes of this definition, "control" includes the right to grant
+patent sublicenses in a manner consistent with the requirements of
+this License.
+
+  Each contributor grants you a non-exclusive, worldwide, royalty-free
+patent license under the contributor's essential patent claims, to
+make, use, sell, offer for sale, import and otherwise run, modify and
+propagate the contents of its contributor version.
+
+  In the following three paragraphs, a "patent license" is any express
+agreement or commitment, however denominated, not to enforce a patent
+(such as an express permission to practice a patent or covenant not to
+sue for patent infringement).  To "grant" such a patent license to a
+party means to make such an agreement or commitment not to enforce a
+patent against the party.
+
+  If you convey a covered work, knowingly relying on a patent license,
+and the Corresponding Source of the work is not available for anyone
+to copy, free of charge and under the terms of this License, through a
+publicly available network server or other readily accessible means,
+then you must either (1) cause the Corresponding Source to be so
+available, or (2) arrange to deprive yourself of the benefit of the
+patent license for this particular work, or (3) arrange, in a manner
+consistent with the requirements of this License, to extend the patent
+license to downstream recipients.  "Knowingly relying" means you have
+actual knowledge that, but for the patent license, your conveying the
+covered work in a country, or your recipient's use of the covered work
+in a country, would infringe one or more identifiable patents in that
+country that you have reason to believe are valid.
+
+  If, pursuant to or in connection with a single transaction or
+arrangement, you convey, or propagate by procuring conveyance of, a
+covered work, and grant a patent license to some of the parties
+receiving the covered work authorizing them to use, propagate, modify
+or convey a specific copy of the covered work, then the patent license
+you grant is automatically extended to all recipients of the covered
+work and works based on it.
+
+  A patent license is "discriminatory" if it does not include within
+the scope of its coverage, prohibits the exercise of, or is
+conditioned on the non-exercise of one or more of the rights that are
+specifically granted under this License.  You may not convey a covered
+work if you are a party to an arrangement with a third party that is
+in the business of distributing software, under which you make payment
+to the third party based on the extent of your activity of conveying
+the work, and under which the third party grants, to any of the
+parties who would receive the covered work from you, a discriminatory
+patent license (a) in connection with copies of the covered work
+conveyed by you (or copies made from those copies), or (b) primarily
+for and in connection with specific products or compilations that
+contain the covered work, unless you entered into that arrangement,
+or that patent license was granted, prior to 28 March 2007.
+
+  Nothing in this License shall be construed as excluding or limiting
+any implied license or other defenses to infringement that may
+otherwise be available to you under applicable patent law.
+
+  12. No Surrender of Others' Freedom.
+
+  If conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot convey a
+covered work so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you may
+not convey it at all.  For example, if you agree to terms that obligate you
+to collect a royalty for further conveying from those to whom you convey
+the Program, the only way you could satisfy both those terms and this
+License would be to refrain entirely from conveying the Program.
+
+  13. Use with the GNU Affero General Public License.
+
+  Notwithstanding any other provision of this License, you have
+permission to link or combine any covered work with a work licensed
+under version 3 of the GNU Affero General Public License into a single
+combined work, and to convey the resulting work.  The terms of this
+License will continue to apply to the part which is the covered work,
+but the special requirements of the GNU Affero General Public License,
+section 13, concerning interaction through a network will apply to the
+combination as such.
+
+  14. Revised Versions of this License.
+
+  The Free Software Foundation may publish revised and/or new versions of
+the GNU General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+  Each version is given a distinguishing version number.  If the
+Program specifies that a certain numbered version of the GNU General
+Public License "or any later version" applies to it, you have the
+option of following the terms and conditions either of that numbered
+version or of any later version published by the Free Software
+Foundation.  If the Program does not specify a version number of the
+GNU General Public License, you may choose any version ever published
+by the Free Software Foundation.
+
+  If the Program specifies that a proxy can decide which future
+versions of the GNU General Public License can be used, that proxy's
+public statement of acceptance of a version permanently authorizes you
+to choose that version for the Program.
+
+  Later license versions may give you additional or different
+permissions.  However, no additional obligations are imposed on any
+author or copyright holder as a result of your choosing to follow a
+later version.
+
+  15. Disclaimer of Warranty.
+
+  THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
+APPLICABLE LAW.  EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
+HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
+OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
+IS WITH YOU.  SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
+ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+  16. Limitation of Liability.
+
+  IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
+THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
+GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
+USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
+DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
+PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
+EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGES.
+
+  17. Interpretation of Sections 15 and 16.
+
+  If the disclaimer of warranty and limitation of liability provided
+above cannot be given local legal effect according to their terms,
+reviewing courts shall apply local law that most closely approximates
+an absolute waiver of all civil liability in connection with the
+Program, unless a warranty or assumption of liability accompanies a
+copy of the Program in return for a fee.
+
+                     END OF TERMS AND CONDITIONS
+
+            How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+state the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    cdist
+    Copyright (C) 2019  ungleich-public
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+Also add information on how to contact you by electronic and paper mail.
+
+  If the program does terminal interaction, make it output a short
+notice like this when it starts in an interactive mode:
+
+    cdist  Copyright (C) 2019  ungleich-public
+    This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License.  Of course, your program's commands
+might be different; for a GUI interface, you would use an "about box".
+
+  You should also get your employer (if you work as a programmer) or school,
+if any, to sign a "copyright disclaimer" for the program, if necessary.
+For more information on this, and how to apply and follow the GNU GPL, see
+<http://www.gnu.org/licenses/>.
+
+  The GNU General Public License does not permit incorporating your program
+into proprietary programs.  If your program is a subroutine library, you
+may consider it more useful to permit linking proprietary applications with
+the library.  If this is what you want to do, use the GNU Lesser General
+Public License instead of this License.  But first, please read
+<http://www.gnu.org/philosophy/why-not-lgpl.html>.

Makefile

@@ -18,31 +18,30 @@
 #
 #
 
-helper=./bin/build-helper
+.PHONY: help
+help:
+	@echo "Please use \`make <target>' where <target> is one of"
+	@echo "man             build only man user documentation"
+	@echo "html            build only html user documentation"
+	@echo "docs            build both man and html user documentation"
+	@echo "dotman          build man pages for types in your ~/.cdist directory"
+	@echo "speeches        build speeches pdf files"
+	@echo "install         install in the system site-packages directory"
+	@echo "install-user    install in the user site-packages directory"
+	@echo "docs-clean      clean documentation"
+	@echo "clean           clean"
 
-DOCS_SRC_DIR=docs/src
-SPEECHDIR=docs/speeches
-TYPEDIR=cdist/conf/type
-
-WEBSRCDIR=docs/web
-
-WEBDIR=$$HOME/vcs/www.nico.schottelius.org
-WEBBLOG=$(WEBDIR)/blog
-WEBBASE=$(WEBDIR)/software/cdist
-WEBPAGE=$(WEBBASE).mdwn
-
-CHANGELOG_VERSION=$(shell $(helper) changelog-version)
-CHANGELOG_FILE=docs/changelog
-
-PYTHON_VERSION=cdist/version.py
+DOCS_SRC_DIR=./docs/src
+SPEECHDIR=./docs/speeches
+TYPEDIR=./cdist/conf/type
 
 SPHINXM=make -C $(DOCS_SRC_DIR) man
 SPHINXH=make -C $(DOCS_SRC_DIR) html
 SPHINXC=make -C $(DOCS_SRC_DIR) clean
+
 ################################################################################
 # Manpages
 #
-MAN1DSTDIR=$(DOCS_SRC_DIR)/man1
 MAN7DSTDIR=$(DOCS_SRC_DIR)/man7
 
 # Manpages #1: Types
@@ -54,6 +53,7 @@ MANTYPES=$(subst /man.rst,.rst,$(MANTYPEPREFIX))
 
 # Link manpage: do not create man.html but correct named file
 $(MAN7DSTDIR)/cdist-type%.rst: $(TYPEDIR)/%/man.rst
+	mkdir -p $(MAN7DSTDIR)
 	ln -sf "../../../$^" $@
 
 # Manpages #2: reference
@@ -63,11 +63,28 @@ DOCSREFSH=$(DOCS_SRC_DIR)/cdist-reference.rst.sh
 $(DOCSREF): $(DOCSREFSH)
 	$(DOCSREFSH)
 
+# Html types list with references
+DOCSTYPESREF=$(MAN7DSTDIR)/cdist-types.rst
+DOCSTYPESREFSH=$(DOCS_SRC_DIR)/cdist-types.rst.sh
+
+$(DOCSTYPESREF): $(DOCSTYPESREFSH)
+	$(DOCSTYPESREFSH)
+
+DOCSCFGSKEL=./configuration/cdist.cfg.skeleton
+
+configskel: $(DOCSCFGSKEL)
+	cp -f "$(DOCSCFGSKEL)" "$(DOCS_SRC_DIR)/"
+
+version:
+	@[ -f "cdist/version.py" ] || { \
+		printf "Missing 'cdist/version.py', please generate it first.\n" && exit 1; \
+	}
+
 # Manpages #3: generic part
-man: $(MANTYPES) $(DOCSREF) $(PYTHON_VERSION)
+man: version configskel $(MANTYPES) $(DOCSREF) $(DOCSTYPESREF)
 	$(SPHINXM)
 
-html: $(MANTYPES) $(DOCSREF) $(PYTHON_VERSION)
+html: version configskel $(MANTYPES) $(DOCSREF) $(DOCSTYPESREF)
 	$(SPHINXH)
 
 docs: man html
@@ -75,24 +92,6 @@ docs: man html
 docs-clean:
 	$(SPHINXC)
 
-# Manpages #5: release part
-MANWEBDIR=$(WEBBASE)/man/$(CHANGELOG_VERSION)
-HTMLBUILDDIR=docs/dist/html
-
-docs-dist: html
-	rm -rf "${MANWEBDIR}"
-	mkdir -p "${MANWEBDIR}"
-	# mkdir -p "${MANWEBDIR}/man1" "${MANWEBDIR}/man7"
-	# cp ${MAN1DSTDIR}/*.html ${MAN1DSTDIR}/*.css ${MANWEBDIR}/man1
-	# cp ${MAN7DSTDIR}/*.html ${MAN7DSTDIR}/*.css ${MANWEBDIR}/man7
-	cp -R ${HTMLBUILDDIR}/* ${MANWEBDIR}
-	cd ${MANWEBDIR} && git add . && git commit -m "cdist manpages update: $(CHANGELOG_VERSION)" || true
-
-man-latest-link: web-pub
-	# Fix ikiwiki, which does not like symlinks for pseudo security
-	ssh staticweb.ungleich.ch \
-		"cd /home/services/www/nico/nico.schottelius.org/www/software/cdist/man/ && rm -f latest && ln -sf "$(CHANGELOG_VERSION)" latest"
-
 # Manpages: .cdist Types
 DOT_CDIST_PATH=${HOME}/.cdist
 DOTMAN7DSTDIR=$(MAN7DSTDIR)
@@ -105,8 +104,7 @@ DOTMANTYPES=$(subst /man.rst,.rst,$(DOTMANTYPEPREFIX))
 $(DOTMAN7DSTDIR)/cdist-type%.rst: $(DOTTYPEDIR)/%/man.rst
 	ln -sf "$^" $@
 
-# Manpages #3: generic part
-dotman: $(DOTMANTYPES)
+dotman: version configskel $(DOTMANTYPES) $(DOCSREF) $(DOCSTYPESREF)
 	$(SPHINXM)
 
 ################################################################################
@@ -114,7 +112,6 @@ dotman: $(DOTMANTYPES)
 #
 SPEECHESOURCES=$(SPEECHDIR)/*.tex
 SPEECHES=$(SPEECHESOURCES:.tex=.pdf)
-SPEECHESWEBDIR=$(WEBBASE)/speeches
 
 # Create speeches and ensure Toc is up-to-date
 $(SPEECHDIR)/%.pdf: $(SPEECHDIR)/%.tex
@@ -124,128 +121,28 @@ $(SPEECHDIR)/%.pdf: $(SPEECHDIR)/%.tex
 
 speeches: $(SPEECHES)
 
-speeches-dist: speeches
-	rm -rf "${SPEECHESWEBDIR}"
-	mkdir -p "${SPEECHESWEBDIR}"
-	cp ${SPEECHES} "${SPEECHESWEBDIR}"
-	cd ${SPEECHESWEBDIR} && git add . && git commit -m "cdist speeches updated" || true
-
 ################################################################################
-# Website
+# Misc
 #
-
-BLOGFILE=$(WEBBLOG)/cdist-$(CHANGELOG_VERSION)-released.mdwn
-
-$(BLOGFILE): $(CHANGELOG_FILE)
-	$(helper) blog $(CHANGELOG_VERSION) $(BLOGFILE)
-
-web-blog: $(BLOGFILE)
-
-web-doc:
-	# Go to top level, because of cdist.mdwn
-	rsync -av "$(WEBSRCDIR)/" "${WEBBASE}/.."
-	cd "${WEBBASE}/.." && git add cdist* && git commit -m "cdist doc update" cdist* || true
-
-web-dist: web-blog web-doc
-
-web-pub: web-dist docs-dist speeches-dist
-	cd "${WEBDIR}" && make pub
-
-web-release-all: man-latest-link
-web-release-all-no-latest: web-pub
-
-################################################################################
-# Release: Mailinglist
-#
-ML_FILE=.lock-ml
-
-# Only send mail once - lock until new changelog things happened
-$(ML_FILE): $(CHANGELOG_FILE)
-	$(helper) ml-release $(CHANGELOG_VERSION)
-	touch $@
-
-ml-release: $(ML_FILE)
-
-
-################################################################################
-# pypi
-#
-PYPI_FILE=.pypi-release
-$(PYPI_FILE): man $(PYTHON_VERSION)
-	python3 setup.py sdist upload
-	touch $@
-
-pypi-release: $(PYPI_FILE)
-################################################################################
-# archlinux
-#
-ARCHLINUX_FILE=.lock-archlinux
-ARCHLINUXTAR=cdist-$(CHANGELOG_VERSION)-1.src.tar.gz
-
-$(ARCHLINUXTAR): PKGBUILD
-	umask 022; mkaurball
-
-PKGBUILD: PKGBUILD.in $(PYTHON_VERSION)
-	./PKGBUILD.in $(CHANGELOG_VERSION)
-
-$(ARCHLINUX_FILE): $(ARCHLINUXTAR) $(PYTHON_VERSION)
-	burp -c system $(ARCHLINUXTAR)
-	touch $@
-
-archlinux-release: $(ARCHLINUX_FILE)
-
-################################################################################
-# Release
-#
-
-$(PYTHON_VERSION) version: .git/refs/heads/master
-	$(helper) version
-
-# Code that is better handled in a shell script
-check-%:
-	$(helper) $@
-
-release:
-	$(helper) $@
-
-################################################################################
-# Cleanup
-#
-
-clean:
+clean: docs-clean
 	rm -f $(DOCS_SRC_DIR)/cdist-reference.rst
+	rm -f $(DOCS_SRC_DIR)/cdist-types.rst
+	rm -f $(DOCS_SRC_DIR)/cdist.cfg.skeleton
 
 	find "$(DOCS_SRC_DIR)" -mindepth 2 -type l \
 	| xargs rm -f
 
-	make -C $(DOCS_SRC_DIR) clean
-
 	find * -name __pycache__  | xargs rm -rf
 
-	# Archlinux
-	rm -f cdist-*.pkg.tar.xz cdist-*.tar.gz
-	rm -rf pkg/ src/
-
-	rm -f MANIFEST PKGBUILD
-	rm -rf dist/
-
-	# Signed release
-	rm -f cdist-*.tar.gz
-	rm -f cdist-*.tar.gz.asc
-
-distclean: clean
-	rm -f cdist/version.py
+	# distutils
+	rm -rf ./build
 
 ################################################################################
-# Misc
+# install
 #
 
-# The pub is Nico's "push to all git remotes" way ("make pub")
-pub:
-	git push --mirror
+install:
+	python3 setup.py install
 
-test:
-	$(helper) $@
-
-pep8:
-	$(helper) $@
+install-user:
+	python3 setup.py install --user

PKGBUILD.in

@@ -9,7 +9,7 @@ pkgver=$version
 pkgrel=1
 pkgdesc='A Usable Configuration Management System"'
 arch=('any')
-url='http://www.nico.schottelius.org/software/cdist/'
+url='https://www.cdi.st/'
 license=('GPL3')
 depends=('python>=3.2.0')
 source=("http://pypi.python.org/packages/source/c/cdist/cdist-\${pkgver}.tar.gz")

README

@@ -1,6 +0,0 @@
-cdist
------
-
-cdist is a usable configuration management system.
-
-For the web documentation have a look at docs/web/.

README-maintainers

@@ -0,0 +1,4 @@
+Maintainers should use ./bin/cdist-build-helper script.
+
+Makefile is intended for end users. It can be used for non-maintaining
+targets that can be run from pure source (without git repository).

README.md

@@ -0,0 +1,31 @@
+# cdist
+
+**cdist** is a usable configuration management system.
+
+It adheres to the [**KISS principle**](https://en.wikipedia.org/wiki/KISS_principle)
+and is being used in small up to enterprise grade environments.
+
+For more information have a look at [**homepage**](https://cdi.st)
+or at **``docs/src``** for manual in **reStructuredText** format.
+
+## Contributing
+
+Merge/Pull requests can be made in both
+[upstream **GitLab**](https://code.ungleich.ch/ungleich-public/cdist/merge_requests)
+(managed by [**ungleich**](https://ungleich.ch))
+and [**GitHub** project](https://github.com/ungleich/cdist/pulls).
+
+Issues can be made and other project management activites happen
+[**only in GitLab**](https://code.ungleich.ch/ungleich-public/cdist)
+(needs [**ungleich** account](https://account.ungleich.ch)).
+
+For community-maintained types there is
+[**cdist-contrib** project](https://code.ungleich.ch/ungleich-public/cdist-contrib).
+
+## Participating
+
+IRC: ``#cdist`` @ freenode
+
+Matrix: ``#cdist:ungleich.ch``
+
+Mattermost: https://chat.ungleich.ch/ungleich/channels/cdist

bin/build-helper

@@ -1,406 +0,0 @@
-#!/bin/sh
-#
-# 2011-2013 Nico Schottelius (nico-cdist at schottelius.org)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
-#
-# This file contains the heavy lifting found usually in the Makefile
-#
-
-basedir=${0%/*}/../
-# Change to checkout directory
-cd "$basedir"
-
-version=$(git describe)
-
-option=$1; shift
-
-case "$option" in
-    changelog-changes)
-        if [ "$#" -eq 1 ]; then
-            start=$1
-        else
-            start="[[:digit:]]"
-        fi
-
-        end="[[:digit:]]"
-
-        awk -F: "BEGIN { start=0 }
-            {
-                if(start == 0) {
-                    if (\$0 ~ /^$start/) {
-                        start = 1
-                    }
-                } else {
-                    if (\$0 ~ /^$end/) {
-                        exit
-                    } else {
-                        print \$0 
-                    }
-                }
-            }" "$basedir/docs/changelog"
-    ;;
-
-    changelog-version)
-        # get version from changelog
-        grep '^[[:digit:]]' "$basedir/docs/changelog" | head -n1 | sed 's/:.*//'
-    ;;
-
-    check-date)
-        # verify date in changelog is today
-        date_today="$(date +%Y-%m-%d)"
-        date_changelog=$(grep '^[[:digit:]]' "$basedir/docs/changelog" | head -n1 | sed 's/.*: //')
-
-        if [ "$date_today" != "$date_changelog" ]; then
-            echo "Date in changelog is not today"
-            echo "Changelog: $date_changelog"
-            exit 1
-        fi
-    ;;
-
-    check-unittest)
-        "$0" test
-    ;;
-
-    blog)
-        version=$1; shift
-        blogfile=$1; shift
-        dir=${blogfile%/*}
-        file=${blogfile##*/}
-
-
-        cat << eof > "$blogfile"
-[[!meta title="Cdist $version released"]]
-
-Here's a short overview about the changes found in version ${version}:
-
-eof
-
-        $0 changelog-changes "$version" >> "$blogfile"
-
-        cat << eof >> "$blogfile"
-For more information visit the [[cdist homepage|software/cdist]].
-
-[[!tag cdist config unix]]
-eof
-        cd "$dir"
-        git add "$file"
-        # Allow git commit to fail if there are no changes
-        git commit -m "cdist blog update: $version" "$blogfile" || true
-    ;;
-
-    ml-release)
-        if [ $# -ne 1 ]; then
-            echo "$0 ml-release version" >&2
-            exit 1
-        fi
-
-        version=$1; shift
-
-        to_a=cdist
-        to_d=l.schottelius.org
-        to=${to_a}@${to_d}
-
-        from_a=nico-cdist
-        from_d=schottelius.org
-        from=${from_a}@${from_d}
-
-        ( 
-        cat << eof
-From: Nico -telmich- Schottelius <$from>
-To: cdist mailing list <$to>
-Subject: cdist $version released
-
-Hello .*,
-
-cdist $version has been released with the following changes:
-
-eof
-
-        "$0" changelog-changes "$version"
-        cat << eof
-
-Cheers,
-
-Nico
-
--- 
-Automatisation at its best level. With cdist.
-eof
-        ) | /usr/sbin/sendmail -f "$from" "$to"
-    ;;
-
-
-    release-git-tag)
-        target_version=$($0 changelog-version)
-        if git rev-parse --verify refs/tags/$target_version 2>/dev/null; then
-            echo "Tag for $target_version exists, aborting"
-            exit 1
-        fi
-        printf "Enter tag description for ${target_version}: "
-        read tagmessage
-
-        # setup for signed tags:
-        # gpg --fulL-gen-key
-        # gpg --list-secret-keys --keyid-format LONG
-        # git config --local user.signingkey <id>
-        # for exporting pub key:
-        #     gpg --armor --export <id> > pubkey.asc
-        #     gpg --output pubkey.gpg --export <id>
-        # show tag with signature
-        # git show <tag>
-        # verify tag signature
-        # git tag -v <tag>
-        #
-        # gpg verify signature
-        # gpg --verify <asc-file> <file>
-        # gpg --no-default-keyring --keyring <pubkey.gpg> --verify <asc-file> <file>
-        # Ensure gpg-agent is running.
-        export GPG_TTY=$(tty)
-        gpg-agent
-
-        git tag -s "$target_version" -m "$tagmessage"
-        git push --tags
-    ;;
-
-    sign-git-release)
-        if [ $# -lt 2 ]
-        then
-            printf "usage: $0 sign-git-release TAG TOKEN [ARCHIVE]\n"
-            printf "    if ARCHIVE is not specified then it is created\n"
-            exit 1
-        fi
-        tag="$1"
-        if ! git rev-parse -q --verify "${tag}" >/dev/null 2>&1
-        then
-            printf "Tag \"${tag}\" not found.\n"
-            exit 1
-        fi
-        token="$2"
-        if [ $# -gt 2 ]
-        then
-            archivename="$3"
-        else
-            archivename="cdist-${tag}.tar.gz"
-            git archive --prefix="cdist-${tag}/" -o "${archivename}" "${tag}" \
-                || exit 1
-        fi
-        gpg --armor --detach-sign "${archivename}" || exit 1
-
-        # make github release
-        curl -H "Authorization: token ${token}" \
-            --request POST \
-            --data "{ \"tag_name\":\"${tag}\", \
-                      \"target_commitish\":\"master\", \
-                      \"name\": \"${tag}\", \
-                      \"body\":\"${tag}\", \
-                      \"draft\":false, \
-                      \"prerelease\": false}" \
-            "https://api.github.com/repos/ungleich/cdist/releases" || exit 1
-
-        # get release ID
-        repoid=$(curl "https://api.github.com/repos/ungleich/cdist/releases/tags/${tag}" \
-            | python3 -c 'import json; import sys; print(json.loads(sys.stdin.read())["id"])') \
-            || exit 1
-
-        # upload archive and then signature
-        curl -H "Authorization: token ${token}" \
-             -H "Accept: application/vnd.github.manifold-preview" \
-             -H "Content-Type: application/x-gtar" \
-             --data-binary @${archivename} \
-            "https://uploads.github.com/repos/ungleich/cdist/releases/${repoid}/assets?name=${archivename}" \
-            || exit 1
-        curl -H "Authorization: token ${token}" \
-             -H "Accept: application/vnd.github.manifold-preview" \
-             -H "Content-Type: application/pgp-signature" \
-             --data-binary @${archivename}.asc \
-            "https://uploads.github.com/repos/ungleich/cdist/releases/${repoid}/assets?name=${archivename}.asc" \
-            || exit 1
-
-        # remove generated files (archive and asc)
-        if [ $# -eq 2]
-        then
-            rm -f "${archivename}"
-        fi
-        rm -f "${archivename}.asc"
-    ;;
-
-    release)
-        set -e
-        target_version=$($0 changelog-version)
-        target_branch=$($0 version-branch)
-
-        echo "Beginning release process for $target_version"
-
-        # First check everything is sane
-        "$0" check-date
-        "$0" check-unittest
-        "$0" check-pep8
-
-        # Generate version file to be included in packaging
-        "$0" target-version
-
-        # Ensure the git status is clean, else abort
-        if ! git diff-index --name-only --exit-code HEAD ; then
-            echo "Unclean tree, see files above, aborting"
-            exit 1
-        fi
-
-        # Ensure we are on the master branch
-        masterbranch=yes
-        if [ "$(git rev-parse --abbrev-ref HEAD)" != "master" ]; then
-            echo "Releases are happening from the master branch, aborting"
-
-            echo "Enter the magic word to release anyway"
-            read magicword
-
-            if [ "$magicword" = "iknowwhatido" ]; then
-                masterbranch=no
-            else
-                exit 1
-            fi
-        fi
-
-        if [ "$masterbranch" = yes ]; then
-            # Ensure version branch exists
-            if ! git rev-parse --verify refs/heads/$target_branch 2>/dev/null; then
-                git branch "$target_branch"
-            fi
-
-            # Merge master branch into version branch
-            git checkout "$target_branch"
-            git merge master
-        fi
-
-        # Verify that after the merge everything works
-        "$0" check-date
-        "$0" check-unittest
-
-        # Generate documentation (man and html)
-        # First, clean old generated docs
-        make docs-clean
-        make docs
-
-        # Generate speeches (indirect check if they build)
-        make speeches
-
-        ############################################################# 
-        # Everything green, let's do the release
-
-        # Tag the current commit
-        "$0" release-git-tag
-
-        # sign git tag
-        printf "Enter github authentication token: "
-        read token
-        "$0" sign-git-release "${target_version}" "${token}"
-
-        # Also merge back the version branch
-        if [ "$masterbranch" = yes ]; then
-            git checkout master
-            git merge "$target_branch"
-        fi
-
-        # Publish git changes
-        make pub
-
-        # publish man, speeches, website
-        if [ "$masterbranch" = yes ]; then
-            make web-release-all
-        else
-            make web-release-all-no-latest
-        fi
-
-        # Ensure that pypi release has the right version
-        "$0" version
-
-        # Create and publish package for pypi
-        make pypi-release
-
-        # Archlinux release is based on pypi
-        make archlinux-release
-
-        # Announce change on ML
-        make ml-release
-
-        cat << eof
-Manual steps post release:
-
-    - linkedin
-    - hackernews
-    - reddit
-    - twitter
-
-eof
-
-    ;;
-
-    test)
-        export PYTHONPATH="$(pwd -P)"
-
-        if [ $# -lt 1 ]; then
-            python3 -m cdist.test
-        else
-            python3 -m unittest "$@"
-        fi
-    ;;
-
-    pep8)
-        pep8 "${basedir}" "${basedir}/scripts/cdist" | less
-    ;;
-
-    check-pep8)
-        "$0" pep8
-        echo "Please review pep8 report."
-        while true
-        do
-            echo "Continue (yes/no)?"
-            any=
-            read any
-            case "$any" in
-                yes)
-                    break
-                ;;
-                no)
-                    exit 1
-                ;;
-                *)
-                    echo "Please answer with 'yes' or 'no' explicitly."
-                ;;
-        esac
-        done
-    ;;
-
-    version-branch)
-        "$0" changelog-version | cut -d. -f '1,2'
-    ;;
-
-    version)
-        echo "VERSION = \"$(git describe)\"" > cdist/version.py
-    ;;
-
-    target-version)
-        target_version=$($0 changelog-version)
-        echo "VERSION = \"${target_version}\"" > cdist/version.py
-    ;;
-
-    *)
-        echo "Unknown helper target $@ - aborting"
-        exit 1
-    ;;
-
-esac

bin/build-helper.freebsd

@@ -1,468 +0,0 @@
-#!/bin/sh
-#
-# 2011-2013 Nico Schottelius (nico-cdist at schottelius.org)
-# 2016 Darko Poljak (darko.poljak at gmail.com)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
-#
-# This file contains the heavy lifting found usually in the Makefile
-#
-
-# vars for make
-helper=$0
-
-basedir=${0%/*}/../
-# run_as is used to check how the script is called (by $0 value)
-# currently supported sufixes for $0 are:
-# .freebsd - run as freebsd
-basename=${0##*/}
-run_as=${basename#*.}
-case "$run_as" in
-    freebsd)
-        to_a=cdist-configuration-management
-        to_d=googlegroups.com
-        from_a=darko.poljak
-        from_d=gmail.com
-        ml_name="Darko Poljak"
-        ml_sig_name="Darko"
-
-        # vars for make
-        WEBDIR=../vcs/www.nico.schottelius.org
-    ;;
-    *)
-        to_a=cdist
-        to_d=l.schottelius.org
-        from_a=nico-cdist
-        from_d=schottelius.org
-        ml_name="Nico -telmich- Schottelius"
-        ml_sig_name="Nico"
-
-        # vars for make
-        WEBDIR=$$HOME/vcs/www.nico.schottelius.org
-    ;;
-esac
-
-# Change to checkout directory
-cd "$basedir"
-
-version=$(git describe)
-
-option=$1; shift
-
-case "$option" in
-    print-make-vars)
-        printf "helper: ${helper}\n"
-        printf "WEBDIR: ${WEBDIR}\n"
-    ;;
-    print-runas)
-        printf "run_as: $run_as\n"
-    ;;
-    changelog-changes)
-        if [ "$#" -eq 1 ]; then
-            start=$1
-        else
-            start="[[:digit:]]"
-        fi
-
-        end="[[:digit:]]"
-
-        awk -F: "BEGIN { start=0 }
-            {
-                if(start == 0) {
-                    if (\$0 ~ /^$start/) {
-                        start = 1
-                    }
-                } else {
-                    if (\$0 ~ /^$end/) {
-                        exit
-                    } else {
-                        print \$0 
-                    }
-                }
-            }" "$basedir/docs/changelog"
-    ;;
-
-    changelog-version)
-        # get version from changelog
-        grep '^[[:digit:]]' "$basedir/docs/changelog" | head -n1 | sed 's/:.*//'
-    ;;
-
-    check-date)
-        # verify date in changelog is today
-        date_today="$(date +%Y-%m-%d)"
-        date_changelog=$(grep '^[[:digit:]]' "$basedir/docs/changelog" | head -n1 | sed 's/.*: //')
-
-        if [ "$date_today" != "$date_changelog" ]; then
-            echo "Date in changelog is not today"
-            echo "Changelog: $date_changelog"
-            exit 1
-        fi
-    ;;
-
-    check-unittest)
-        "$0" test
-    ;;
-
-    blog)
-        version=$1; shift
-        blogfile=$1; shift
-        dir=${blogfile%/*}
-        file=${blogfile##*/}
-
-
-        cat << eof > "$blogfile"
-[[!meta title="Cdist $version released"]]
-
-Here's a short overview about the changes found in version ${version}:
-
-eof
-
-        $0 changelog-changes "$version" >> "$blogfile"
-
-        cat << eof >> "$blogfile"
-For more information visit the [[cdist homepage|software/cdist]].
-
-[[!tag cdist config unix]]
-eof
-        cd "$dir"
-        git add "$file"
-        # Allow git commit to fail if there are no changes
-        git commit -m "cdist blog update: $version" "$blogfile" || true
-    ;;
-
-    ml-release)
-        if [ $# -ne 1 ]; then
-            echo "$0 ml-release version" >&2
-            exit 1
-        fi
-
-        version=$1; shift
-
-        to=${to_a}@${to_d}
-        from=${from_a}@${from_d}
-
-        ( 
-        cat << eof
-From: ${ml_name} <$from>
-To: cdist mailing list <$to>
-Subject: cdist $version released
-
-Hello .*,
-
-cdist $version has been released with the following changes:
-
-eof
-
-        "$0" changelog-changes "$version"
-        cat << eof
-
-Cheers,
-
-${ml_sig_name}
-
--- 
-Automatisation at its best level. With cdist.
-eof
-        ) | /usr/sbin/sendmail -f "$from" "$to"
-    ;;
-
-    release-git-tag)
-        target_version=$($0 changelog-version)
-        if git rev-parse --verify refs/tags/$target_version 2>/dev/null; then
-            echo "Tag for $target_version exists, aborting"
-            exit 1
-        fi
-        printf "Enter tag description for ${target_version}: "
-        read tagmessage
-
-        # setup for signed tags:
-        # gpg --fulL-gen-key
-        # gpg --list-secret-keys --keyid-format LONG
-        # git config --local user.signingkey <id>
-        # for exporting pub key:
-        #     gpg --armor --export <id> > pubkey.asc
-        #     gpg --output pubkey.gpg --export <id>
-        # show tag with signature
-        # git show <tag>
-        # verify tag signature
-        # git tag -v <tag>
-        #
-        # gpg verify signature
-        # gpg --verify <asc-file> <file>
-        # gpg --no-default-keyring --keyring <pubkey.gpg> --verify <asc-file> <file>
-        # Ensure gpg-agent is running.
-        export GPG_TTY=$(tty)
-        gpg-agent
-
-        git tag -s "$target_version" -m "$tagmessage"
-        git push --tags
-    ;;
-
-    sign-git-release)
-        if [ $# -lt 2 ]
-        then
-            printf "usage: $0 sign-git-release TAG TOKEN [ARCHIVE]\n"
-            printf "    if ARCHIVE is not specified then it is created\n"
-            exit 1
-        fi
-        tag="$1"
-        if ! git rev-parse -q --verify "${tag}" >/dev/null 2>&1
-        then
-            printf "Tag \"${tag}\" not found.\n"
-            exit 1
-        fi
-        token="$2"
-        if [ $# -gt 2 ]
-        then
-            archivename="$3"
-        else
-            archivename="cdist-${tag}.tar.gz"
-            git archive --prefix="cdist-${tag}/" -o "${archivename}" "${tag}" \
-                || exit 1
-        fi
-        gpg --armor --detach-sign "${archivename}" || exit 1
-
-        # make github release
-        curl -H "Authorization: token ${token}" \
-            --request POST \
-            --data "{ \"tag_name\":\"${tag}\", \
-                      \"target_commitish\":\"master\", \
-                      \"name\": \"${tag}\", \
-                      \"body\":\"${tag}\", \
-                      \"draft\":false, \
-                      \"prerelease\": false}" \
-            "https://api.github.com/repos/ungleich/cdist/releases" || exit 1
-
-        # get release ID
-        repoid=$(curl "https://api.github.com/repos/ungleich/cdist/releases/tags/${tag}" \
-            | python3 -c 'import json; import sys; print(json.loads(sys.stdin.read())["id"])') \
-            || exit 1
-
-        # upload archive and then signature
-        curl -H "Authorization: token ${token}" \
-             -H "Accept: application/vnd.github.manifold-preview" \
-             -H "Content-Type: application/x-gtar" \
-             --data-binary @${archivename} \
-            "https://uploads.github.com/repos/ungleich/cdist/releases/${repoid}/assets?name=${archivename}" \
-            || exit 1
-        curl -H "Authorization: token ${token}" \
-             -H "Accept: application/vnd.github.manifold-preview" \
-             -H "Content-Type: application/pgp-signature" \
-             --data-binary @${archivename}.asc \
-            "https://uploads.github.com/repos/ungleich/cdist/releases/${repoid}/assets?name=${archivename}.asc" \
-            || exit 1
-
-        # remove generated files (archive and asc)
-        if [ $# -eq 2]
-        then
-            rm -f "${archivename}"
-        fi
-        rm -f "${archivename}.asc"
-    ;;
-
-    release)
-        set -e
-        target_version=$($0 changelog-version)
-        target_branch=$($0 version-branch)
-
-        echo "Beginning release process for $target_version"
-
-        # First check everything is sane
-        "$0" check-date
-        "$0" check-unittest
-        "$0" check-pep8
-
-        # Generate version file to be included in packaging
-        "$0" target-version
-
-        # Ensure the git status is clean, else abort
-        if ! git diff-index --name-only --exit-code HEAD ; then
-            echo "Unclean tree, see files above, aborting"
-            exit 1
-        fi
-
-        # Ensure we are on the master branch
-        masterbranch=yes
-        if [ "$(git rev-parse --abbrev-ref HEAD)" != "master" ]; then
-            echo "Releases are happening from the master branch, aborting"
-
-            echo "Enter the magic word to release anyway"
-            read magicword
-
-            if [ "$magicword" = "iknowwhatido" ]; then
-                masterbranch=no
-            else
-                exit 1
-            fi
-        fi
-
-        if [ "$masterbranch" = yes ]; then
-            # Ensure version branch exists
-            if ! git rev-parse --verify refs/heads/$target_branch 2>/dev/null; then
-                git branch "$target_branch"
-            fi
-
-            # Merge master branch into version branch
-            git checkout "$target_branch"
-            git merge master
-        fi
-
-        # Verify that after the merge everything works
-        "$0" check-date
-        "$0" check-unittest
-
-        # Generate documentation (man and html)
-        # First, clean old generated docs
-        make helper=${helper} WEBDIR=${WEBDIR} docs-clean
-        make helper=${helper} WEBDIR=${WEBDIR} docs
-
-        # Generate speeches (indirect check if they build)
-        make helper=${helper} WEBDIR=${WEBDIR} speeches
-
-        ############################################################# 
-        # Everything green, let's do the release
-
-        # Tag the current commit
-        "$0" release-git-tag
-
-        # sign git tag
-        printf "Enter github authentication token: "
-        read token
-        "$0" sign-git-release "${target_version}" "${token}"
-
-        # Also merge back the version branch
-        if [ "$masterbranch" = yes ]; then
-            git checkout master
-            git merge "$target_branch"
-        fi
-
-        # Publish git changes
-        case "$run_as" in
-            freebsd)
-                # if we are not Nico :) then just push, no mirror
-                git push
-                # push also new branch and set up tracking
-                git push -u origin "${target_branch}"
-            ;;
-            *)
-                make helper=${helper} WEBDIR=${WEBDIR} pub
-            ;;
-        esac
-
-        # publish man, speeches, website
-        if [ "$masterbranch" = yes ]; then
-            make helper=${helper} WEBDIR=${WEBDIR} web-release-all
-        else
-            make helper=${helper} WEBDIR=${WEBDIR} web-release-all-no-latest
-        fi
-
-        # Ensure that pypi release has the right version
-        "$0" version
-
-        # Create and publish package for pypi
-        make helper=${helper} WEBDIR=${WEBDIR} pypi-release
-
-        case "$run_as" in
-            freebsd)
-            ;;
-            *)
-                # Archlinux release is based on pypi
-                make archlinux-release
-            ;;
-        esac
-
-        # Announce change on ML
-        make helper=${helper} WEBDIR=${WEBDIR} ml-release
-
-        cat << eof
-Manual steps post release:
-
-    - linkedin
-    - hackernews
-    - reddit
-    - twitter
-
-eof
-
-        case "$run_as" in
-            freebsd)
-                cat <<eof
-Additional steps post release:
-    - archlinux release
-eof
-            ;;
-            *)
-            ;;
-        esac
-
-    ;;
-
-    test)
-        export PYTHONPATH="$(pwd -P)"
-
-        if [ $# -lt 1 ]; then
-            python3 -m cdist.test
-        else
-            python3 -m unittest "$@"
-        fi
-    ;;
-
-    pep8)
-        pep8 "${basedir}" "${basedir}/scripts/cdist" | less
-    ;;
-
-    check-pep8)
-        "$0" pep8
-        echo "Please review pep8 report."
-        while true
-        do
-            echo "Continue (yes/no)?"
-            any=
-            read any
-            case "$any" in
-                yes)
-                    break
-                ;;
-                no)
-                    exit 1
-                ;;
-                *)
-                    echo "Please answer with 'yes' or 'no' explicitly."
-                ;;
-        esac
-        done
-    ;;
-
-    version-branch)
-        "$0" changelog-version | cut -d. -f '1,2'
-    ;;
-
-    version)
-        echo "VERSION = \"$(git describe)\"" > cdist/version.py
-    ;;
-
-    target-version)
-        target_version=$($0 changelog-version)
-        echo "VERSION = \"${target_version}\"" > cdist/version.py
-    ;;
-
-    *)
-        echo "Unknown helper target $@ - aborting"
-        exit 1
-    ;;
-
-esac

bin/cdist

@@ -1,7 +1,8 @@
-#!/bin/sh
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 #
-# 2012 Nico Schottelius (nico-cdist at schottelius.org)
+# 2010-2016 Nico Schottelius (nico-cdist at schottelius.org)
+# 2016 Darko Poljak (darko.poljak at gmail.com)
 #
 # This file is part of cdist.
 #
@@ -20,14 +21,81 @@
 #
 #
 
-# Wrapper for real script to allow execution from checkout
-dir=${0%/*}
+import logging
+import os
+import sys
 
-# Ensure version is present - the bundled/shipped version contains a static version,
-# the git version contains a dynamic version
-"$dir/build-helper" version
+# See if this file's parent is cdist module
+# and if so add it to module search path.
+cdist_dir = os.path.realpath(
+    os.path.join(
+        os.path.dirname(os.path.realpath(__file__)),
+        os.pardir))
+cdist_init_dir = os.path.join(cdist_dir, 'cdist', '__init__.py')
+if os.path.exists(cdist_init_dir):
+    sys.path.insert(0, cdist_dir)
 
-libdir=$(cd "${dir}/../" && pwd -P)
-export PYTHONPATH="${libdir}"
+import cdist            # noqa 402
+import cdist.argparse   # noqa 402
+import cdist.banner     # noqa 402
+import cdist.config     # noqa 402
+import cdist.install    # noqa 402
+import cdist.shell      # noqa 402
+import cdist.inventory  # noqa 402
 
-"$dir/../scripts/cdist" "$@"
+
+def commandline():
+    """Parse command line"""
+
+    # preos subcommand hack
+    if len(sys.argv) > 1 and sys.argv[1] == 'preos':
+        return cdist.preos.PreOS.commandline(sys.argv[1:])
+    parser, cfg = cdist.argparse.parse_and_configure(sys.argv[1:])
+    args = cfg.get_args()
+
+    # Work around python 3.3 bug:
+    # http://bugs.python.org/issue16308
+    # http://bugs.python.org/issue9253
+
+    # FIXME: catching AttributeError also hides
+    # real problems.. try a different way
+
+    # FIXME: we always print main help, not
+    # the help of the actual parser being used!
+    try:
+        getattr(args, "func")
+    except AttributeError:
+        parser['main'].print_help()
+        sys.exit(0)
+
+    args.func(args)
+
+
+if __name__ == "__main__":
+    if sys.version < cdist.MIN_SUPPORTED_PYTHON_VERSION:
+        print('Python >= {} is required on the source host.'.format(
+                cdist.MIN_SUPPORTED_PYTHON_VERSIO), file=sys.stderr)
+        sys.exit(1)
+
+    exit_code = 0
+
+    try:
+        import re
+        import os
+
+        if re.match("__", os.path.basename(sys.argv[0])):
+            import cdist.emulator
+            emulator = cdist.emulator.Emulator(sys.argv)
+            emulator.run()
+        else:
+            commandline()
+
+    except KeyboardInterrupt:
+        exit_code = 2
+
+    except cdist.Error as e:
+        log = logging.getLogger("cdist")
+        log.error(e)
+        exit_code = 1
+
+    sys.exit(exit_code)

bin/cdist-build-helper

@@ -0,0 +1,574 @@
+#!/bin/sh
+#
+# 2011-2013 Nico Schottelius (nico-cdist at schottelius.org)
+# 2016-2019 Darko Poljak (darko.poljak at gmail.com)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+# This file contains the heavy lifting found usually in the Makefile.
+#
+
+usage() {
+    printf "usage: %s TARGET [TARGET-ARGS...]
+    Available targets:
+        changelog-changes
+        changelog-version
+        check-date
+        check-unittest
+        ml-release
+        archlinux-release
+        pypi-release
+        release-git-tag
+        sign-git-release
+        release
+        test
+        test-remote
+        pycodestyle
+        pep8
+        check-pycodestyle
+        shellcheck-global-explorers
+        shellcheck-type-explorers
+        shellcheck-manifests
+        shellcheck-local-gencodes
+        shellcheck-remote-gencodes
+        shellcheck-bin
+        shellcheck-gencodes
+        shellcheck-types
+        shellcheck
+        shellcheck-type-files
+        shellcheck-with-files
+        shellcheck-build-helper
+        check-shellcheck
+        version-branch
+        version
+        target-version
+        clean
+        distclean\n" "$1"
+}
+
+basename="${0##*/}"
+
+if [ $# -lt 1 ]
+then
+    usage "${basename}"
+    exit 1
+fi
+
+option=$1; shift
+
+SHELLCHECKCMD="shellcheck -s sh -f gcc -x"
+# Skip SC2154 for variables starting with __ since such variables are cdist
+# environment variables.
+SHELLCHECK_SKIP=': __.*is referenced but not assigned.*\[SC2154\]'
+SHELLCHECKTMP=".shellcheck.tmp"
+
+# Change to checkout directory
+basedir="${0%/*}/../"
+cd "$basedir"
+
+case "$option" in
+    changelog-changes)
+        if [ "$#" -eq 1 ]; then
+            start=$1
+        else
+            start="[[:digit:]]"
+        fi
+
+        end="[[:digit:]]"
+
+        awk -F: "BEGIN { start=0 }
+            {
+                if(start == 0) {
+                    if (\$0 ~ /^$start/) {
+                        start = 1
+                    }
+                } else {
+                    if (\$0 ~ /^$end/) {
+                        exit
+                    } else {
+                        print \$0
+                    }
+                }
+            }" "$basedir/docs/changelog"
+    ;;
+
+    changelog-version)
+        # get version from changelog
+        grep '^[[:digit:]]' "$basedir/docs/changelog" | head -n1 | sed 's/:.*//'
+    ;;
+
+    check-date)
+        # verify date in changelog is today
+        date_today="$(date +%Y-%m-%d)"
+        date_changelog=$(grep '^[[:digit:]]' "$basedir/docs/changelog" | head -n1 | sed 's/.*: //')
+
+        if [ "$date_today" != "$date_changelog" ]; then
+            printf "Date in changelog is not today\n"
+            printf "Changelog date: %s\n" "${date_changelog}"
+            exit 1
+        fi
+    ;;
+
+    check-unittest)
+        "$0" test
+    ;;
+
+    ml-release)
+        if [ $# -ne 1 ]; then
+            printf "%s ml-release version\n" "$0" >&2
+            exit 1
+        fi
+
+        version=$1; shift
+
+        (
+        cat << eof
+Subject: cdist $version has been released
+
+Hello .*,
+
+cdist $version has been released with the following changes:
+
+eof
+
+        "$0" changelog-changes "$version"
+        cat << eof
+
+eof
+        ) > mailinglist.tmp
+    ;;
+
+    archlinux-release)
+        if [ $# -ne 1 ]; then
+            printf "%s archlinux-release version\n" "$0" >&2
+            exit 1
+        fi
+        version=$1; shift
+
+        ARCHLINUXTAR="cdist-${version}-1.src.tar.gz"
+        ./PKGBUILD.in "${version}"
+        umask 022
+        mkaurball
+        burp -c system "${ARCHLINUXTAR}"
+    ;;
+
+    pypi-release)
+        # Ensure that pypi release has the right version
+        "$0" version
+
+        make docs-clean
+        make docs
+        python3 setup.py sdist upload
+    ;;
+
+    release-git-tag)
+        target_version=$($0 changelog-version)
+        if git rev-parse --verify "refs/tags/${target_version}" 2>/dev/null; then
+            printf "Tag for %s exists, aborting\n" "${target_version}"
+            exit 1
+        fi
+        printf "Enter tag description for %s: " "${target_version}"
+        read -r tagmessage
+
+        # setup for signed tags:
+        # gpg --fulL-gen-key
+        # gpg --list-secret-keys --keyid-format LONG
+        # git config --local user.signingkey <id>
+        # for exporting pub key:
+        #     gpg --armor --export <id> > pubkey.asc
+        #     gpg --output pubkey.gpg --export <id>
+        # show tag with signature
+        # git show <tag>
+        # verify tag signature
+        # git tag -v <tag>
+        #
+        # gpg verify signature
+        # gpg --verify <asc-file> <file>
+        # gpg --no-default-keyring --keyring <pubkey.gpg> --verify <asc-file> <file>
+        # Ensure gpg-agent is running.
+        GPG_TTY=$(tty)
+        export GPG_TTY
+        gpg-agent
+
+        git tag -s "$target_version" -m "$tagmessage"
+        git push --tags
+    ;;
+
+    sign-git-release)
+        if [ $# -lt 2 ]
+        then
+            printf "usage: %s sign-git-release TAG TOKEN [ARCHIVE]\n" "$0"
+            printf "    if ARCHIVE is not specified then it is created\n"
+            exit 1
+        fi
+        tag="$1"
+        if ! git rev-parse -q --verify "${tag}" >/dev/null 2>&1
+        then
+            printf "Tag \"%s\" not found.\n" "${tag}"
+            exit 1
+        fi
+        token="$2"
+        if [ $# -gt 2 ]
+        then
+            archivename="$3"
+        else
+            archivename="cdist-${tag}.tar"
+            git archive --prefix="cdist-${tag}/" -o "${archivename}" "${tag}" \
+                || exit 1
+            # make sure target version is generated
+            "$0" target-version
+            tar -x -f "${archivename}" || exit 1
+            cp cdist/version.py "cdist-${tag}/cdist/version.py" || exit 1
+            tar -c -f "${archivename}" "cdist-${tag}/" || exit 1
+            rm -r -f "cdist-${tag}/"
+            gzip "${archivename}" || exit 1
+            archivename="${archivename}.gz"
+        fi
+        gpg --armor --detach-sign "${archivename}" || exit 1
+
+        project="ungleich-public%2Fcdist"
+        sed_cmd='s/^.*"markdown":"\([^"]*\)".*$/\1/'
+
+        # upload archive
+        response_archive=$(curl -f -X POST \
+             --http1.1 \
+             -H "PRIVATE-TOKEN: ${token}" \
+             -F "file=@${archivename}" \
+             "https://code.ungleich.ch/api/v4/projects/${project}/uploads" \
+             | sed "${sed_cmd}") || exit 1
+
+        # upload archive signature
+        response_archive_sig=$(curl -f -X POST \
+             --http1.1 \
+             -H "PRIVATE-TOKEN: ${token}" \
+             -F "file=@${archivename}.asc" \
+            "https://code.ungleich.ch/api/v4/projects/${project}/uploads" \
+             | sed "${sed_cmd}") || exit 1
+
+        # make release
+        changelog=$("$0" changelog-changes "$1" | sed 's/^[[:space:]]*//')
+        release_notes=$(
+            printf "%s\n\n%s\n\n**Changelog**\n\n%s\n" \
+                "${response_archive}" "${response_archive_sig}" "${changelog}"
+        )
+        curl -f -X POST \
+             -H "PRIVATE-TOKEN: ${token}" \
+             -F "description=${release_notes}" \
+            "https://code.ungleich.ch/api/v4/projects/${project}/repository/tags/${tag}/release" \
+            || exit 1
+
+        # remove generated files (archive and asc)
+        if [ $# -eq 2 ]
+        then
+            rm -f "${archivename}"
+        fi
+        rm -f "${archivename}.asc"
+    ;;
+
+    release)
+        set -e
+        target_version=$($0 changelog-version)
+        target_branch=$($0 version-branch)
+
+        printf "Beginning release process for %s\n" "${target_version}"
+
+        # First check everything is sane
+        "$0" check-date
+        "$0" check-unittest
+        "$0" check-pycodestyle
+        "$0" check-shellcheck
+
+        # Generate version file to be included in packaging
+        "$0" target-version
+
+        # Ensure the git status is clean, else abort
+        if ! git diff-index --name-only --exit-code HEAD ; then
+            printf "Unclean tree, see files above, aborting.\n"
+            exit 1
+        fi
+
+        # Ensure we are on the master branch
+        masterbranch=yes
+        if [ "$(git rev-parse --abbrev-ref HEAD)" != "master" ]; then
+            printf "Releases are happening from the master branch, aborting.\n"
+
+            printf "Enter the magic word to release anyway:"
+            read -r magicword
+
+            if [ "$magicword" = "iknowwhatido" ]; then
+                masterbranch=no
+            else
+                exit 1
+            fi
+        fi
+
+        if [ "$masterbranch" = yes ]; then
+            # Ensure version branch exists
+            if ! git rev-parse --verify "refs/heads/${target_branch}" 2>/dev/null; then
+                git branch "$target_branch"
+            fi
+
+            # Merge master branch into version branch
+            git checkout "$target_branch"
+            git merge master
+        fi
+
+        # Verify that after the merge everything works
+        "$0" check-date
+        "$0" check-unittest
+
+        # Generate documentation (man and html)
+        # First, clean old generated docs
+        make docs-clean
+        make docs
+
+        #############################################################
+        # Everything green, let's do the release
+
+        # Tag the current commit
+        "$0" release-git-tag
+
+        # Also merge back the version branch
+        if [ "$masterbranch" = yes ]; then
+            git checkout master
+            git merge "$target_branch"
+        fi
+
+        # Publish git changes
+        # if you want to have mirror locally then uncomment this and comment below
+        #     git push --mirror
+            git push
+            # push also new branch and set up tracking
+            git push -u origin "${target_branch}"
+        # fi
+
+        # Create and publish package for pypi
+        "$0" pypi-release
+
+        # sign git tag
+        printf "Enter upstream repository authentication token: "
+        read -r token
+        "$0" sign-git-release "${target_version}" "${token}"
+
+        # Announce change on ML
+        "$0" ml-release "${target_version}"
+
+        cat << eof
+Manual steps post release:
+    - cdist-web
+    - send generated mailinglist.tmp mail
+eof
+    ;;
+
+    test)
+        if [ ! -f "cdist/version.py" ]
+        then
+            printf "cdist/version.py is missing, generate it first.\n"
+            exit 1
+        fi
+
+        PYTHONPATH="$(pwd -P)"
+        export PYTHONPATH
+
+        if [ $# -lt 1 ]; then
+            python3 -m cdist.test
+        else
+            python3 -m unittest "$@"
+        fi
+    ;;
+
+    test-remote)
+        if [ ! -f "cdist/version.py" ]
+        then
+            printf "cdist/version.py is missing, generate it first.\n"
+            exit 1
+        fi
+
+        PYTHONPATH="$(pwd -P)"
+        export PYTHONPATH
+
+        python3 -m cdist.test.exec.remote
+    ;;
+
+    pycodestyle|pep8)
+        pycodestyle "${basedir}" "${basedir}/bin/cdist"
+    ;;
+
+    check-pycodestyle)
+        "$0" pycodestyle
+        printf "\\nPlease review pycodestyle report.\\n"
+        while true
+        do
+            printf "Continue (yes/no)?\n"
+            any=
+            read -r any
+            case "$any" in
+                yes)
+                    break
+                ;;
+                no)
+                    exit 1
+                ;;
+                *)
+                    printf "Please answer with 'yes' or 'no' explicitly.\n"
+                ;;
+        esac
+        done
+    ;;
+
+    shellcheck-global-explorers)
+        # shellcheck disable=SC2086
+        find cdist/conf/explorer -type f -exec ${SHELLCHECKCMD} {} + | grep -v "${SHELLCHECK_SKIP}" > "${SHELLCHECKTMP}"
+        test ! -s "${SHELLCHECKTMP}" || { cat "${SHELLCHECKTMP}"; exit 1; }
+    ;;
+
+    shellcheck-type-explorers)
+        # shellcheck disable=SC2086
+        find cdist/conf/type -type f -path "*/explorer/*" -exec ${SHELLCHECKCMD} {} + | grep -v "${SHELLCHECK_SKIP}" > "${SHELLCHECKTMP}"
+        test ! -s "${SHELLCHECKTMP}" || { cat "${SHELLCHECKTMP}"; exit 1; }
+    ;;
+
+    shellcheck-manifests)
+        # shellcheck disable=SC2086
+        find cdist/conf/type -type f -name manifest -exec ${SHELLCHECKCMD} {} + | grep -v "${SHELLCHECK_SKIP}" > "${SHELLCHECKTMP}"
+        test ! -s "${SHELLCHECKTMP}" || { cat "${SHELLCHECKTMP}"; exit 1; }
+    ;;
+
+    shellcheck-local-gencodes)
+        # shellcheck disable=SC2086
+        find cdist/conf/type -type f -name gencode-local -exec ${SHELLCHECKCMD} {} + | grep -v "${SHELLCHECK_SKIP}" > "${SHELLCHECKTMP}"
+        test ! -s "${SHELLCHECKTMP}" || { cat "${SHELLCHECKTMP}"; exit 1; }
+    ;;
+
+    shellcheck-remote-gencodes)
+        # shellcheck disable=SC2086
+        find cdist/conf/type -type f -name gencode-remote -exec ${SHELLCHECKCMD} {} + | grep -v "${SHELLCHECK_SKIP}" > "${SHELLCHECKTMP}"
+        test ! -s "${SHELLCHECKTMP}" || { cat "${SHELLCHECKTMP}"; exit 1; }
+    ;;
+
+    # NOTE: shellcheck-scripts is kept for compatibility
+    shellcheck-bin|shellcheck-scripts)
+        # shellcheck disable=SC2086
+        ${SHELLCHECKCMD} bin/cdist-dump bin/cdist-new-type > "${SHELLCHECKTMP}"
+        test ! -s "${SHELLCHECKTMP}" || { cat "${SHELLCHECKTMP}"; exit 1; }
+    ;;
+
+    shellcheck-gencodes)
+        errors=false
+        "$0" shellcheck-local-gencodes || errors=true
+        "$0" shellcheck-remote-gencodes || errors=true
+        ! $errors || exit 1
+    ;;
+
+    shellcheck-types)
+        errors=false
+        "$0" shellcheck-type-explorers || errors=true
+        "$0" shellcheck-manifests || errors=true
+        "$0" shellcheck-gencodes || errors=true
+        ! $errors || exit 1
+    ;;
+
+    shellcheck)
+        errors=false
+        "$0" shellcheck-global-explorers || errors=true
+        "$0" shellcheck-types || errors=true
+        "$0" shellcheck-bin || errors=true
+        ! $errors || exit 1
+    ;;
+
+    shellcheck-type-files)
+        # shellcheck disable=SC2086
+        find cdist/conf/type -type f -path "*/files/*" -exec ${SHELLCHECKCMD} {} + | grep -v "${SHELLCHECK_SKIP}" > "${SHELLCHECKTMP}"
+        test ! -s "${SHELLCHECKTMP}" || { cat "${SHELLCHECKTMP}"; exit 1; }
+    ;;
+
+    shellcheck-with-files)
+        errors=false
+        "$0" shellcheck || errors=true
+        "$0" shellcheck-type-files || errors=true
+        ! $errors || exit 1
+    ;;
+
+    shellcheck-build-helper)
+        ${SHELLCHECKCMD} ./bin/cdist-build-helper
+    ;;
+
+    check-shellcheck)
+        "$0" shellcheck
+        printf "\\nPlease review shellcheck report.\\n"
+        while true
+        do
+            printf "Continue (yes/no)?\n"
+            any=
+            read -r any
+            case "$any" in
+                yes)
+                    break
+                ;;
+                no)
+                    exit 1
+                ;;
+                *)
+                    printf "Please answer with 'yes' or 'no' explicitly.\n"
+                ;;
+        esac
+        done
+    ;;
+
+    version-branch)
+        "$0" changelog-version | cut -d. -f '1,2'
+    ;;
+
+    version)
+        printf "VERSION = \"%s\"\n" "$(git describe)" > cdist/version.py
+    ;;
+
+    target-version)
+        target_version=$($0 changelog-version)
+        printf "VERSION = \"%s\"\n" "${target_version}" > cdist/version.py
+    ;;
+
+    clean)
+        make clean
+
+        # Archlinux
+        rm -f cdist-*.pkg.tar.xz cdist-*.tar.gz
+        rm -rf pkg/ src/
+
+        rm -f MANIFEST PKGBUILD
+        rm -rf dist/
+
+        # Signed release
+        rm -f cdist-*.tar.gz
+        rm -f cdist-*.tar.gz.asc
+
+        # Temp files
+        rm -f ./*.tmp
+        rm -f ./.*.tmp
+    ;;
+
+    distclean)
+        "$0" clean
+        rm -f cdist/version.py
+    ;;
+    *)
+        printf "Unknown target: '%s'.\n" "${option}" >&2
+        usage "${basename}"
+        exit 1
+    ;;
+
+esac

bin/cdist-dump

@@ -0,0 +1,326 @@
+#!/bin/sh
+
+VERSION="0.0.1"
+RELEASE=""
+
+set -u
+# set -x
+
+hosts=
+cache_dir=~/.cdist/cache
+
+do_all=1
+do_global_explorer=
+do_type_explorer=
+do_script_stdout=
+do_script_stderr=
+do_gencode=
+do_code=
+do_messages=
+do_parameter=
+delimiter=':'
+ln=
+filename_prefix=1
+verbose=0
+
+myname=${0##*/}
+
+print_version()
+{
+    printf "%s %s %s\n" "${myname}" "${VERSION}" "${RELEASE}"
+}
+
+usage()
+{
+    cat << eof
+${myname}: [options] [host...]
+eof
+
+    print_version
+
+    cat << eof
+
+Dump data from cache directories.
+
+host      
+    Dump data for specified hosts. If not specified then all data
+    from cache directory is dumped.
+
+Options
+ -a             dump all
+ -C CACHE-DIR   use specified CACHE-DIR (default: ~/.cdist/cache)
+ -c             dump code-*
+ -d DELIMITER   delimiter used for filename and line number prefix (default: ':')
+ -E             dump global explorers
+ -e             dump type explorers
+ -F             disable filename prefix (enabled by default)
+ -f             enable filename prefix (default)
+ -g             dump gencode-*
+ -h             show this help screen and exit
+ -L             disable line number prefix (default)
+ -l             enable line number prefix (disabled by default)
+ -m             dump messages
+ -o             dump executions' stdout
+ -p             dump parameters
+ -r             dump executions' stderr
+ -V             show version and exit
+ -v             increase verbosity
+eof
+}
+
+exit_err()
+{
+    printf "%s\n" "$1"
+    exit 1
+}
+
+# parse options
+while [ "$#" -ge 1 ]
+do
+    case "$1" in
+        -a)
+            do_all=1
+            ;;
+        -C)
+            if [ "$#" -ge 2 ]
+            then
+                case "$2" in
+                    -*)
+                        exit_err "Missing cache directory"
+                        ;;
+                    *)
+                        cache_dir="$2"
+                        shift
+                        ;;
+                esac
+            else
+                exit_err "Missing cache directory"
+            fi
+            ;;
+        -c)
+            do_code=1
+            do_all=
+            ;;
+        -d)
+            if [ "$#" -ge 2 ]
+            then
+                case "$2" in
+                    -*)
+                        exit_err "Missing delimiter"
+                        ;;
+                    *)
+                        delimiter="$2"
+                        shift
+                        ;;
+                esac
+            else
+                exit_err "Missing delimiter"
+            fi
+            ;;
+        -E)
+            do_global_explorer=1
+            do_all=
+            ;;
+        -e)
+            do_type_explorer=1
+            do_all=
+            ;;
+        -F)
+            filename_prefix=
+            ;;
+        -f)
+            filename_prefix=1
+            ;;
+        -g)
+            do_gencode=1
+            do_all=
+            ;;
+        -h)
+            usage
+            exit 0
+            ;;
+        -L)
+            ln=
+            ;;
+        -l)
+            ln=1
+            ;;
+        -m)
+            do_messages=1
+            do_all=
+            ;;
+        -o)
+            do_script_stdout=1
+            do_all=
+            ;;
+        -p)
+            do_parameter=1
+            do_all=
+            ;;
+        -r)
+            do_script_stderr=1
+            do_all=
+            ;;
+        -V)
+            print_version
+            exit 0
+            ;;
+        -v)
+            verbose=$((verbose + 1))
+            ;;
+        *)
+            hosts="${hosts} $1"
+            break
+            ;;
+    esac
+    shift
+done
+
+if [ "${ln}" = "1" ]
+then
+    ln="NR \"${delimiter}\""
+fi
+
+if [ "${filename_prefix}" = "1" ]
+then
+    filename_prefix="{}${delimiter}"
+fi
+
+if [ "${do_all}" = "1" ]
+then
+    do_global_explorer=1
+    do_type_explorer=1
+    do_script_stdout=1
+    do_script_stderr=1
+    do_gencode=1
+    do_code=1
+    do_messages=1
+    do_parameter=1
+fi
+
+set -- -size +0
+set -- "$@" \(
+or=
+
+print_verbose()
+{
+    if [ "${verbose}" -ge "$1" ]
+    then
+        printf "%s\n" "$2"
+    fi
+}
+
+hor_line()
+{
+    if [ $# -gt 0 ]
+    then
+        c="$1"
+    else
+        c='='
+    fi
+    printf "%78s\n" "" | tr ' ' "${c}"
+}
+
+if [ "${do_global_explorer}" ]
+then
+    print_verbose 2 "Dumping global explorers"
+    # shellcheck disable=SC2086
+    set -- "$@" ${or} \( \
+                -path "*/explorer/*" -a \
+                ! -path "*/conf/*" -a \
+                ! -path "*/object/*/explorer/*" \
+            \)
+    or="-o"
+fi
+
+if [ "${do_type_explorer}" ]
+then
+    print_verbose 2 "Dumping type explorers"
+    set -- "$@" ${or} -path "*/object/*/explorer/*"
+    or="-o"
+fi
+
+if [ "${do_script_stdout}" ]
+then
+    print_verbose 2 "Dumping execution's stdout"
+    set -- "$@" ${or} -path "*/stdout/*"
+    or="-o"
+fi
+
+if [ "${do_script_stderr}" ]
+then
+    print_verbose 2 "Dumping execution's stderr"
+    set -- "$@" ${or} -path "*/stderr/*"
+    or="-o"
+fi
+
+if [ "${do_gencode}" ]
+then
+    print_verbose 2 "Dumping gencode-*"
+    set -- "$@" ${or} \( -name "gencode-*" -a ! -path "*/stdout/*" -a ! -path "*/stderr/*" \)
+    or="-o"
+fi
+
+if [ "${do_code}" ]
+then
+    print_verbose 2 "Dumping code-*"
+    set -- "$@" ${or} \( -name "code-*" -a ! -path "*/stdout/*" -a ! -path "*/stderr/*" \)
+    or="-o"
+fi
+
+if [ "${do_messages}" ]
+then
+    print_verbose 2 "Dumping messages"
+    set -- "$@" ${or} -name "messages"
+    or="-o"
+fi
+
+if [ "${do_parameter}" ]
+then
+    print_verbose 2 "Dumping parameters"
+    set -- "$@" ${or} -path "*/parameter/*"
+    or="-o"
+fi
+
+set -- "$@" \)
+set -- '.' "$@" -exec awk -v prefix="${filename_prefix}" "{print prefix ${ln} \$0}" {} \;
+
+# printf "+ %s\n" "$*"
+
+print_verbose 2 "Using cache dir: ${cache_dir}"
+
+OLD_PWD=$(pwd)
+cd "${cache_dir}" || exit
+
+# If no host is specified then search all.
+[ -z "${hosts}" ] && hosts="-"
+
+for host in ${hosts}
+do
+    [ "${host}" = "-" ] && host=
+    # find host cache directory
+    host_dir=$(find . -name target_host -exec grep -l "${host}" {} +)
+    print_verbose 3 "found host directory files:"
+    print_verbose 3 "${host_dir}"
+
+    OLD_IFS="${IFS}"
+    IFS="
+    "
+
+    for d in ${host_dir}
+    do
+        dir=$(dirname "${d}")
+
+        print_verbose 0 "target host: $(cat "${dir}/target_host"), host directory: ${dir}"
+        hor_line '='
+
+        PREV_PWD=$(pwd)
+        cd "${dir}" || exit
+        # set -x
+        find "$@"
+        # set +x
+        cd "${PREV_PWD}" || exit
+    done
+    IFS="${OLD_IFS}"
+done
+cd "${OLD_PWD}" || exit

bin/cdist-new-type

@@ -0,0 +1,159 @@
+#!/bin/sh
+
+basename="${0##*/}"
+
+if [ $# -lt 3 ]
+then
+    printf "usage: %s TYPE-NAME AUTHOR-NAME AUTHOR-EMAIL [TYPE-BASE-PATH]
+    TYPE-NAME       Name of the type.
+    AUTHOR-NAME     Type author's full name.
+    AUTHOR-EMAIL    Type author's email.
+    TYPE-BASE-PATH  Path to the base directory of the type. If not set it defaults
+                    to '\$PWD/type'.\n" "${basename}"
+    exit 1
+fi
+
+type_name="$1"
+shift
+author_name="$1"
+shift
+author_email="$1"
+shift
+
+if [ $# -ge 1 ]
+then
+    type_base_path="$1"
+    shift
+else
+    #type_base_path=~/.cdist/type
+    type_base_path="$PWD/type"
+fi
+
+error() {
+   printf "%s\n" "$*" >&2
+}
+
+die() {
+   error "$@"
+   exit 1
+}
+
+cd "$type_base_path" || die "Could not change to type directory: $type_base_path.
+You have to specify type base path or run me from within a cdist conf directory,
+e.g. ~/.cdist."
+
+year=$(date +%Y)
+copyright="# $year $author_name ($author_email)"
+
+license="# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+"
+
+set -e
+
+mkdir "$type_name"
+cd "$type_name"
+
+### man page
+header="cdist-type${type_name}(7)"
+header_length="${#header}"
+cat >> man.rst << DONE
+$header
+$(while [ "${header_length}" -gt 0 ]; do printf "="; header_length=$((header_length - 1)); done; printf "\n";)
+
+NAME
+----
+cdist-type${type_name} - TODO
+
+
+DESCRIPTION
+-----------
+This space intentionally left blank.
+
+
+REQUIRED PARAMETERS
+-------------------
+None.
+
+
+OPTIONAL PARAMETERS
+-------------------
+None.
+
+
+BOOLEAN PARAMETERS
+------------------
+None.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    # TODO
+    ${type_name}
+
+
+SEE ALSO
+--------
+:strong:\`TODO\`\\ (7)
+
+
+AUTHORS
+-------
+$author_name <$author_email>
+
+
+COPYING
+-------
+Copyright \(C) $year $author_name. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+DONE
+
+### manifest
+cat >> manifest << DONE
+#!/bin/sh -e
+#
+${copyright}
+#
+${license}
+
+os=\$(cat "\$__global/explorer/os")
+
+case "\$os" in
+   *)
+      printf "Your operating system (%s) is currently not supported by this type (%s)\n" "\$os" "\${__type##*/}" >&2
+      printf "Please contribute an implementation for it if you can.\n" >&2
+      exit 1
+   ;;
+esac
+DONE
+chmod +x manifest
+
+# gencode-remote
+cat >> gencode-remote << DONE
+#!/bin/sh -e
+#
+${copyright}
+#
+${license}
+DONE
+chmod +x gencode-remote
+
+printf "%s/%s\n" "$type_base_path" "$type_name"

cdist/__init__.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 #
 # 2010-2015 Nico Schottelius (nico-cdist at schottelius.org)
+# 2012-2017 Steven Armstrong (steven-cdist at armstrong.cc)
 #
 # This file is part of cdist.
 #
@@ -21,11 +22,27 @@
 
 import os
 import hashlib
+import subprocess
 
 import cdist.log
-import cdist.version
 
-VERSION = cdist.version.VERSION
+
+VERSION = 'unknown version'
+
+try:
+    import cdist.version
+    VERSION = cdist.version.VERSION
+except ModuleNotFoundError:
+    cdist_dir = os.path.abspath(
+        os.path.join(os.path.dirname(__file__), os.pardir))
+    if os.path.isdir(os.path.join(cdist_dir, '.git')):
+        try:
+            VERSION = subprocess.check_output(
+                ['git', 'describe', '--always'],
+                cwd=cdist_dir,
+                universal_newlines=True)
+        except Exception:
+            pass
 
 BANNER = """
              ..          .       .x+=:.        s
@@ -42,11 +59,14 @@ BANNER = """
    "P'        ""         ""
 """
 
-REMOTE_COPY = "scp -o User=root"
+REMOTE_COPY = "scp -o User=root -q"
 REMOTE_EXEC = "ssh -o User=root"
 REMOTE_CMDS_CLEANUP_PATTERN = "ssh -o User=root -O exit -S {}"
 
 
+MIN_SUPPORTED_PYTHON_VERSION = '3.5'
+
+
 class Error(Exception):
     """Base exception class for this project"""
     pass
@@ -80,18 +100,159 @@ class CdistBetaRequired(cdist.Error):
         return err_msg.format(*fmt_args)
 
 
-class CdistObjectError(Error):
-    """Something went wrong with an object"""
+class CdistEntityError(Error):
+    """Something went wrong while executing cdist entity"""
+    def __init__(self, entity_name, entity_params, stdout_paths,
+                 stderr_paths, subject=''):
+        self.entity_name = entity_name
+        self.entity_params = entity_params
+        self.stderr_paths = stderr_paths
+        self.stdout_paths = stdout_paths
+        if isinstance(subject, Error):
+            self.original_error = subject
+        else:
+            self.original_error = None
+        self.message = str(subject)
 
-    def __init__(self, cdist_object, message):
-        self.name = cdist_object.name
-        self.source = " ".join(cdist_object.source)
-        self.message = message
+    def _stdpath(self, stdpaths, header_name):
+        result = {}
+        for name, path in stdpaths:
+            if name not in result:
+                result[name] = []
+            try:
+                if os.path.exists(path) and os.path.getsize(path) > 0:
+                    output = []
+                    label_begin = name + ":" + header_name
+                    output.append(label_begin)
+                    output.append('\n')
+                    output.append('-' * len(label_begin))
+                    output.append('\n')
+                    with open(path, 'r') as fd:
+                        output.append(fd.read())
+                    output.append('\n')
+                    result[name].append(''.join(output))
+            except UnicodeError as ue:
+                result[name].append(('Cannot output {}:{} due to: {}.\n'
+                                     'You can try to read the error file "{}"'
+                                     ' yourself.').format(
+                                         name, header_name, ue, path))
+        return result
+
+    def _stderr(self):
+        return self._stdpath(self.stderr_paths, 'stderr')
+
+    def _stdout(self):
+        return self._stdpath(self.stdout_paths, 'stdout')
+
+    def _update_dict_list(self, target, source):
+        for x in source:
+            if x not in target:
+                target[x] = []
+            target[x].extend(source[x])
+
+    @property
+    def std_streams(self):
+        std_dict = {}
+        self._update_dict_list(std_dict, self._stdout())
+        self._update_dict_list(std_dict, self._stderr())
+        return std_dict
 
     def __str__(self):
-        return '%s: %s (defined at %s)' % (self.name,
-                                           self.message,
-                                           self.source)
+        output = []
+        output.append(self.message)
+        output.append('\n\n')
+        header = "Error processing " + self.entity_name
+        under_header = '=' * len(header)
+        output.append(header)
+        output.append('\n')
+        output.append(under_header)
+        output.append('\n')
+        for param_name, param_value in self.entity_params:
+            output.append(param_name + ': ' + str(param_value))
+            output.append('\n')
+        output.append('\n')
+        for x in self.std_streams:
+            output.append(''.join(self.std_streams[x]))
+        return ''.join(output)
+
+
+class CdistObjectError(CdistEntityError):
+    """Something went wrong while working on a specific cdist object"""
+    def __init__(self, cdist_object, subject=''):
+        params = [
+            ('name', cdist_object.name, ),
+            ('path', cdist_object.absolute_path, ),
+            ('source', " ".join(cdist_object.source), ),
+            ('type', os.path.realpath(
+                cdist_object.cdist_type.absolute_path), ),
+        ]
+        stderr_paths = []
+        for stderr_name in os.listdir(cdist_object.stderr_path):
+            stderr_path = os.path.join(cdist_object.stderr_path,
+                                       stderr_name)
+            stderr_paths.append((stderr_name, stderr_path, ))
+        stdout_paths = []
+        for stdout_name in os.listdir(cdist_object.stdout_path):
+            stdout_path = os.path.join(cdist_object.stdout_path,
+                                       stdout_name)
+            stdout_paths.append((stdout_name, stdout_path, ))
+        super().__init__("object '{}'".format(cdist_object.name),
+                         params, stdout_paths, stderr_paths, subject)
+
+
+class CdistObjectExplorerError(CdistEntityError):
+    """
+    Something went wrong while working on a specific
+    cdist object explorer
+    """
+    def __init__(self, cdist_object, explorer_name, explorer_path,
+                 stderr_path, subject=''):
+        params = [
+            ('object name', cdist_object.name, ),
+            ('object path', cdist_object.absolute_path, ),
+            ('object source', " ".join(cdist_object.source), ),
+            ('object type', os.path.realpath(
+                cdist_object.cdist_type.absolute_path), ),
+            ('explorer name', explorer_name, ),
+            ('explorer path', explorer_path, ),
+        ]
+        stdout_paths = []
+        stderr_paths = [
+            ('remote', stderr_path, ),
+        ]
+        super().__init__("explorer '{}' of object '{}'".format(
+            explorer_name, cdist_object.name), params, stdout_paths,
+            stderr_paths, subject)
+
+
+class InitialManifestError(CdistEntityError):
+    """Something went wrong while executing initial manifest"""
+    def __init__(self, initial_manifest, stdout_path, stderr_path, subject=''):
+        params = [
+            ('path', initial_manifest, ),
+        ]
+        stdout_paths = [
+            ('init', stdout_path, ),
+        ]
+        stderr_paths = [
+            ('init', stderr_path, ),
+        ]
+        super().__init__('initial manifest', params, stdout_paths,
+                         stderr_paths, subject)
+
+
+class GlobalExplorerError(CdistEntityError):
+    """Something went wrong while executing global explorer"""
+    def __init__(self, name, path, stderr_path, subject=''):
+        params = [
+            ('name', name, ),
+            ('path', path, ),
+        ]
+        stderr_paths = [
+            ('remote', stderr_path, ),
+        ]
+        super().__init__("global explorer '{}'".format(name),
+                         params, [], stderr_paths, subject)
 
 
 def file_to_list(filename):
@@ -119,6 +280,11 @@ def str_hash(s):
 
 def home_dir():
     if 'HOME' in os.environ:
-        return os.path.join(os.environ['HOME'], ".cdist")
+        home = os.environ['HOME']
+        if home:
+            rv = os.path.join(home, ".cdist")
+        else:
+            rv = None
     else:
-        return None
+        rv = None
+    return rv

cdist/argparse.py

@@ -3,22 +3,28 @@ import cdist
 import multiprocessing
 import logging
 import collections
+import functools
 import cdist.configuration
+import cdist.log
+import cdist.preos
+import cdist.info
+import cdist.scan.commandline
 
 
 # set of beta sub-commands
-BETA_COMMANDS = set(('install', 'inventory', ))
+BETA_COMMANDS = set(('install', 'inventory', 'scan', ))
 # set of beta arguments for sub-commands
 BETA_ARGS = {
-    'config': set(('jobs', 'tag', 'all_tagged_hosts', 'use_archiving', )),
+    'config': set(('tag', 'all_tagged_hosts', 'use_archiving', )),
 }
-EPILOG = "Get cdist at http://www.nico.schottelius.org/software/cdist/"
+EPILOG = "Get cdist at https://code.ungleich.ch/ungleich-public/cdist"
 # Parser others can reuse
 parser = None
 
 
 _verbosity_level_off = -2
 _verbosity_level = {
+    None: logging.WARNING,
     _verbosity_level_off: logging.OFF,
     -1: logging.ERROR,
     0: logging.WARNING,
@@ -72,15 +78,15 @@ def check_beta(args_dict):
                     raise cdist.CdistBetaRequired(cmd, arg)
 
 
-def check_positive_int(value):
+def check_lower_bounded_int(value, lower_bound, name):
     try:
         val = int(value)
     except ValueError:
         raise argparse.ArgumentTypeError(
                 "{} is invalid int value".format(value))
-    if val <= 0:
+    if val < lower_bound:
         raise argparse.ArgumentTypeError(
-                "{} is invalid positive int value".format(val))
+                "{} is invalid {} value".format(val, name))
     return val
 
 
@@ -94,6 +100,17 @@ def get_parsers():
         parser = {}
     # Options _all_ parsers have in common
     parser['loglevel'] = argparse.ArgumentParser(add_help=False)
+    parser['loglevel'].add_argument(
+            '-l', '--log-level', metavar='LOGLEVEL',
+            type=functools.partial(check_lower_bounded_int, lower_bound=-1,
+                                   name="log level"),
+            help=('Set the specified verbosity level. '
+                  'The levels, in order from the lowest to the highest, are: '
+                  'ERROR (-1), WARNING (0), INFO (1), VERBOSE (2), DEBUG (3), '
+                  'TRACE (4 or higher). If used along with -v then -v '
+                  'increases last set value and -l overwrites last set '
+                  'value.'),
+            action='store', dest='verbose', required=False)
     parser['loglevel'].add_argument(
             '-q', '--quiet',
             help='Quiet mode: disables logging, including WARNING and ERROR.',
@@ -105,19 +122,28 @@ def get_parsers():
                   'is 0 which includes ERROR and WARNING levels. '
                   'The levels, in order from the lowest to the highest, are: '
                   'ERROR (-1), WARNING (0), INFO (1), VERBOSE (2), DEBUG (3) '
-                  'TRACE (4 or higher).'),
-            action='count', default=0)
+                  'TRACE (4 or higher). If used along with -l then -l '
+                  'overwrites last set value and -v increases last set '
+                  'value.'),
+            action='count', default=None)
+
+    parser['colored_output'] = argparse.ArgumentParser(add_help=False)
+    parser['colored_output'].add_argument(
+            '--colors', metavar='WHEN',
+            help="Colorize cdist's output based on log level; "
+                 "WHEN is 'always', 'never', or 'auto'.",
+            action='store', dest='colored_output', required=False,
+            choices=cdist.configuration.ColoredOutputOption.CHOICES)
 
     parser['beta'] = argparse.ArgumentParser(add_help=False)
     parser['beta'].add_argument(
            '-b', '--beta',
            help=('Enable beta functionality. '),
-           action='store_true', dest='beta',
-           default=False)
+           action='store_true', dest='beta', default=None)
 
     # Main subcommand parser
     parser['main'] = argparse.ArgumentParser(
-            description='cdist ' + cdist.VERSION, parents=[parser['loglevel']])
+            description='cdist ' + cdist.VERSION)
     parser['main'].add_argument(
             '-V', '--version', help='Show version.', action='version',
             version='%(prog)s ' + cdist.VERSION)
@@ -148,6 +174,16 @@ def get_parsers():
 
     # Config
     parser['config_main'] = argparse.ArgumentParser(add_help=False)
+    parser['config_main'].add_argument(
+           '-4', '--force-ipv4',
+           help=('Force to use IPv4 addresses only. No influence for custom'
+                 ' remote commands.'),
+           action='store_const', dest='force_ipv', const=4)
+    parser['config_main'].add_argument(
+           '-6', '--force-ipv6',
+           help=('Force to use IPv6 addresses only. No influence for custom'
+                 ' remote commands.'),
+           action='store_const', dest='force_ipv', const=6)
     parser['config_main'].add_argument(
             '-C', '--cache-path-pattern',
             help=('Specify custom cache path pattern. If '
@@ -164,19 +200,31 @@ def get_parsers():
            dest='manifest', required=False)
     parser['config_main'].add_argument(
            '-j', '--jobs', nargs='?',
-           type=check_positive_int,
+           type=functools.partial(check_lower_bounded_int, lower_bound=1,
+                                  name="positive int"),
            help=('Operate in parallel in specified maximum number of jobs. '
                  'Global explorers, object prepare and object run are '
-                 'supported. Without argument CPU count is used by default. '
-                 'Currently in beta.'),
+                 'supported. Without argument CPU count is used by default. '),
            action='store', dest='jobs',
            const=multiprocessing.cpu_count())
+    parser['config_main'].add_argument(
+           '--log-server',
+           action='store_true',
+           help=('Start a log server for sub processes to use. '
+                 'This is mainly useful when running cdist nested '
+                 'from a code-local script. Log server is alwasy '
+                 'implicitly started for \'install\' command.'))
     parser['config_main'].add_argument(
            '-n', '--dry-run',
            help='Do not execute code.', action='store_true')
     parser['config_main'].add_argument(
            '-o', '--out-dir',
            help='Directory to save cdist output in.', dest="out_path")
+    parser['config_main'].add_argument(
+           '-P', '--timestamp',
+           help=('Timestamp log messages with the current local date and time '
+                 'in the format: YYYYMMDDHHMMSS.us.'),
+           action='store_true', dest='timestamp')
     parser['config_main'].add_argument(
            '-R', '--use-archiving', nargs='?',
            choices=('tar', 'tgz', 'tbz2', 'txz',),
@@ -206,6 +254,10 @@ def get_parsers():
                  '(should behave like ssh).'),
            action='store', dest='remote_exec',
            default=None)
+    parser['config_main'].add_argument(
+           '-S', '--disable-saving-output-streams',
+           help='Disable saving output streams.',
+           action='store_false', dest='save_output_streams', default=True)
 
     # Config
     parser['config_args'] = argparse.ArgumentParser(add_help=False)
@@ -218,18 +270,16 @@ def get_parsers():
              help=('List hosts that have all specified tags, '
                    'if -t/--tag is specified.'),
              action="store_true", dest="has_all_tags", default=False)
-    parser['config_args'].add_argument(
-            'host', nargs='*', help='Host(s) to operate on.')
     parser['config_args'].add_argument(
             '-f', '--file',
             help=('Read specified file for a list of additional hosts to '
                   'operate on or if \'-\' is given, read stdin (one host per '
-                  'line). If no host or host file is specified then, by '
-                  'default, read hosts from stdin.'),
+                  'line).'),
             dest='hostfile', required=False)
     parser['config_args'].add_argument(
            '-p', '--parallel', nargs='?', metavar='HOST_MAX',
-           type=check_positive_int,
+           type=functools.partial(check_lower_bounded_int, lower_bound=1,
+                                  name="positive int"),
            help=('Operate on multiple hosts in parallel for specified maximum '
                  'hosts at a time. Without argument CPU count is used by '
                  'default.'),
@@ -245,8 +295,11 @@ def get_parsers():
                    'list all hosts that contain any of specified tags. '
                    'Currently in beta.'),
              dest='tag', required=False, action="store_true", default=False)
+    parser['config_args'].add_argument(
+            'host', nargs='*', help='Host(s) to operate on.')
     parser['config'] = parser['sub'].add_parser(
             'config', parents=[parser['loglevel'], parser['beta'],
+                               parser['colored_output'],
                                parser['common'],
                                parser['config_main'],
                                parser['inventory_common'],
@@ -259,15 +312,13 @@ def get_parsers():
     parser['install'].set_defaults(func=cdist.install.Install.commandline)
 
     # Inventory
-    parser['inventory'] = parser['sub'].add_parser(
-           'inventory', parents=[parser['loglevel'], parser['beta'],
-                                 parser['common'],
-                                 parser['inventory_common']])
+    parser['inventory'] = parser['sub'].add_parser('inventory')
     parser['invsub'] = parser['inventory'].add_subparsers(
             title="Inventory commands", dest="subcommand")
 
     parser['add-host'] = parser['invsub'].add_parser(
             'add-host', parents=[parser['loglevel'], parser['beta'],
+                                 parser['colored_output'],
                                  parser['common'],
                                  parser['inventory_common']])
     parser['add-host'].add_argument(
@@ -275,13 +326,12 @@ def get_parsers():
     parser['add-host'].add_argument(
            '-f', '--file',
            help=('Read additional hosts to add from specified file '
-                 'or from stdin if \'-\' (each host on separate line). '
-                 'If no host or host file is specified then, by default, '
-                 'read from stdin.'),
+                 'or from stdin if \'-\' (each host on separate line). '),
            dest='hostfile', required=False)
 
     parser['add-tag'] = parser['invsub'].add_parser(
             'add-tag', parents=[parser['loglevel'], parser['beta'],
+                                parser['colored_output'],
                                 parser['common'],
                                 parser['inventory_common']])
     parser['add-tag'].add_argument(
@@ -290,20 +340,12 @@ def get_parsers():
     parser['add-tag'].add_argument(
            '-f', '--file',
            help=('Read additional hosts to add tags from specified file '
-                 'or from stdin if \'-\' (each host on separate line). '
-                 'If no host or host file is specified then, by default, '
-                 'read from stdin. If no tags/tagfile nor hosts/hostfile'
-                 ' are specified then tags are read from stdin and are'
-                 ' added to all hosts.'),
+                 'or from stdin if \'-\' (each host on separate line). '),
            dest='hostfile', required=False)
     parser['add-tag'].add_argument(
            '-T', '--tag-file',
            help=('Read additional tags to add from specified file '
-                 'or from stdin if \'-\' (each tag on separate line). '
-                 'If no tag or tag file is specified then, by default, '
-                 'read from stdin. If no tags/tagfile nor hosts/hostfile'
-                 ' are specified then tags are read from stdin and are'
-                 ' added to all hosts.'),
+                 'or from stdin if \'-\' (each tag on separate line). '),
            dest='tagfile', required=False)
     parser['add-tag'].add_argument(
            '-t', '--taglist',
@@ -313,6 +355,7 @@ def get_parsers():
 
     parser['del-host'] = parser['invsub'].add_parser(
             'del-host', parents=[parser['loglevel'], parser['beta'],
+                                 parser['colored_output'],
                                  parser['common'],
                                  parser['inventory_common']])
     parser['del-host'].add_argument(
@@ -323,13 +366,12 @@ def get_parsers():
     parser['del-host'].add_argument(
             '-f', '--file',
             help=('Read additional hosts to delete from specified file '
-                  'or from stdin if \'-\' (each host on separate line). '
-                  'If no host or host file is specified then, by default, '
-                  'read from stdin.'),
+                  'or from stdin if \'-\' (each host on separate line). '),
             dest='hostfile', required=False)
 
     parser['del-tag'] = parser['invsub'].add_parser(
             'del-tag', parents=[parser['loglevel'], parser['beta'],
+                                parser['colored_output'],
                                 parser['common'],
                                 parser['inventory_common']])
     parser['del-tag'].add_argument(
@@ -342,20 +384,13 @@ def get_parsers():
     parser['del-tag'].add_argument(
             '-f', '--file',
             help=('Read additional hosts to delete tags for from specified '
-                  'file or from stdin if \'-\' (each host on separate line). '
-                  'If no host or host file is specified then, by default, '
-                  'read from stdin. If no tags/tagfile nor hosts/hostfile'
-                  ' are specified then tags are read from stdin and are'
-                  ' deleted from all hosts.'),
+                  'file or from stdin if \'-\' (each host on separate '
+                  'line). '),
             dest='hostfile', required=False)
     parser['del-tag'].add_argument(
             '-T', '--tag-file',
             help=('Read additional tags from specified file '
-                  'or from stdin if \'-\' (each tag on separate line). '
-                  'If no tag or tag file is specified then, by default, '
-                  'read from stdin. If no tags/tagfile nor'
-                  ' hosts/hostfile are specified then tags are read from'
-                  ' stdin and are added to all hosts.'),
+                  'or from stdin if \'-\' (each tag on separate line). '),
             dest='tagfile', required=False)
     parser['del-tag'].add_argument(
             '-t', '--taglist',
@@ -365,6 +400,7 @@ def get_parsers():
 
     parser['list'] = parser['invsub'].add_parser(
             'list', parents=[parser['loglevel'], parser['beta'],
+                             parser['colored_output'],
                              parser['common'],
                              parser['inventory_common']])
     parser['list'].add_argument(
@@ -392,15 +428,78 @@ def get_parsers():
     parser['inventory'].set_defaults(
             func=cdist.inventory.Inventory.commandline)
 
+    # PreOS
+    parser['preos'] = parser['sub'].add_parser('preos', add_help=False)
+
     # Shell
     parser['shell'] = parser['sub'].add_parser(
-            'shell', parents=[parser['loglevel']])
+            'shell', parents=[parser['loglevel'], parser['colored_output']])
     parser['shell'].add_argument(
             '-s', '--shell',
             help=('Select shell to use, defaults to current shell. Used shell'
                   ' should be POSIX compatible shell.'))
     parser['shell'].set_defaults(func=cdist.shell.Shell.commandline)
 
+    # Info
+    parser['info'] = parser['sub'].add_parser('info')
+    parser['info'].add_argument(
+            '-a', '--all', help='Display all info. This is the default.',
+            action='store_true', default=False)
+    parser['info'].add_argument(
+            '-c', '--conf-dir',
+            help='Add configuration directory (can be repeated).',
+            action='append')
+    parser['info'].add_argument(
+            '-e', '--global-explorers',
+            help='Display info for global explorers.', action='store_true',
+            default=False)
+    parser['info'].add_argument(
+            '-F', '--fixed-string',
+            help='Interpret pattern as a fixed string.', action='store_true',
+            default=False)
+    parser['info'].add_argument(
+            '-f', '--full', help='Display full details.',
+            action='store_true', default=False)
+    parser['info'].add_argument(
+           '-g', '--config-file',
+           help='Use specified custom configuration file.',
+           dest="config_file", required=False)
+    parser['info'].add_argument(
+            '-t', '--types', help='Display info for types.',
+            action='store_true', default=False)
+    parser['info'].add_argument(
+            'pattern', nargs='?', help='Glob pattern.')
+    parser['info'].set_defaults(func=cdist.info.Info.commandline)
+
+    # Scan = config + further
+    parser['scan'] = parser['sub'].add_parser('scan', add_help=False,
+                                              parents=[parser['config']])
+
+    parser['scan'] = parser['sub'].add_parser(
+            'scan', parents=[parser['loglevel'],
+                             parser['beta'],
+                             parser['colored_output'],
+                             parser['common'],
+                             parser['config_main']])
+
+    parser['scan'].add_argument(
+        '-m', '--mode', help='Which modes should run',
+        action='append', default=[],
+        choices=['scan', 'trigger'])
+    parser['scan'].add_argument(
+        '--config',
+        action='store_true',
+        help='Try to configure detected hosts')
+    parser['scan'].add_argument(
+        '-I', '--interfaces',
+        action='append',  default=[],
+        help='On which interfaces to scan/trigger')
+    parser['scan'].add_argument(
+        '-d', '--delay',
+        action='store',  default=3600,
+        help='How long to wait before reconfiguring after last try')
+    parser['scan'].set_defaults(func=cdist.scan.commandline.commandline)
+
     for p in parser:
         parser[p].epilog = EPILOG
 
@@ -408,19 +507,29 @@ def get_parsers():
 
 
 def handle_loglevel(args):
-    if args.quiet:
+    if hasattr(args, 'quiet') and args.quiet:
         args.verbose = _verbosity_level_off
 
-    logging.root.setLevel(_verbosity_level[args.verbose])
+    logging.getLogger().setLevel(_verbosity_level[args.verbose])
+
+
+def handle_log_colors(args):
+    if cdist.configuration.ColoredOutputOption.translate(args.colored_output):
+        cdist.log.CdistFormatter.USE_COLORS = True
 
 
 def parse_and_configure(argv, singleton=True):
     parser = get_parsers()
     parser_args = parser['main'].parse_args(argv)
-    cfg = cdist.configuration.Configuration(parser_args, singleton=singleton)
-    args = cfg.get_args()
+    try:
+        cfg = cdist.configuration.Configuration(parser_args,
+                                                singleton=singleton)
+        args = cfg.get_args()
+    except ValueError as e:
+        raise cdist.Error(str(e))
     # Loglevels are handled globally in here
     handle_loglevel(args)
+    handle_log_colors(args)
 
     log = logging.getLogger("cdist")
 

cdist/conf/explorer/cpu_cores

@@ -25,13 +25,22 @@
 os=$("$__explorer/os")
 case "$os" in
     "macosx")
-        echo "$(sysctl -n hw.physicalcpu)"
+        sysctl -n hw.physicalcpu
+    ;;
+
+    "openbsd")
+        sysctl -n hw.ncpuonline
+    ;;
+
+    "freebsd"|"netbsd")
+        PATH=$(getconf PATH)
+        sysctl -n hw.ncpu
     ;;
 
     *)
         if [ -r /proc/cpuinfo ]; then
             cores="$(grep "core id" /proc/cpuinfo | sort | uniq | wc -l)"
-            if [ ${cores} -eq 0 ]; then
+            if [ "${cores}" -eq 0 ]; then
                 cores="1"
             fi
             echo "$cores"

cdist/conf/explorer/cpu_sockets

@@ -25,14 +25,14 @@
 os=$("$__explorer/os")
 case "$os" in
     "macosx")
-        echo "$(system_profiler SPHardwareDataType | grep "Number of Processors" | awk -F': ' '{print $2}')"
+        system_profiler SPHardwareDataType | grep "Number of Processors" | awk -F': ' '{print $2}'
     ;;
 
     *)
     if [ -r /proc/cpuinfo ]; then
-        sockets="$(grep "physical id" /proc/cpuinfo | sort | uniq | wc -l)"
-        if [ ${sockets} -eq 0 ]; then
-            sockets="$(cat /proc/cpuinfo | grep "processor" | wc -l)"
+        sockets="$(grep "physical id" /proc/cpuinfo | sort -u | wc -l)"
+        if [ "${sockets}" -eq 0 ]; then
+            sockets="$(grep -c "processor" /proc/cpuinfo)"
         fi
         echo "${sockets}"
     fi

cdist/conf/explorer/disks

@@ -1,2 +1,66 @@
-cd /dev
-echo sd? hd? vd?
+#!/bin/sh -e
+#
+# based on previous work by other people, modified by:
+# 2020 Dennis Camera <dennis.camera at ssrq-sds-fds.ch>
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+# Finds disks of the system (excl. ram disks, floppy, cdrom)
+
+uname_s="$(uname -s)"
+
+case $uname_s in
+    FreeBSD)
+        sysctl -n kern.disks
+    ;;
+    OpenBSD)
+        sysctl -n hw.disknames | grep -Eo '[lsw]d[0-9]+'
+    ;;
+    NetBSD)
+        PATH=$(getconf PATH)
+        sysctl -n hw.disknames | awk -v RS=' ' '/^[lsw]d[0-9]+/'
+    ;;
+    Linux)
+        # list of major device numbers toexclude:
+        #  ram disks, floppies, cdroms
+        # https://www.kernel.org/doc/Documentation/admin-guide/devices.txt
+        ign_majors='1 2 11'
+
+        if command -v lsblk >/dev/null 2>&1
+        then
+            lsblk -e "$(echo "$ign_majors" | tr ' ' ',')" -dno name
+        elif test -d /sys/block/
+        then
+            # shellcheck disable=SC2012
+            ls -1 /sys/block/ \
+            | awk -v ign_majors="$(echo "$ign_majors" | tr ' ' '|')" '
+                {
+                  devfile = "/sys/block/" $0 "/dev"
+                  getline devno < devfile
+                  close(devfile)
+                  if (devno !~ "^(" ign_majors "):") print
+                }'
+        else
+            echo "Don't know how to list disks on Linux without lsblk and sysfs." >&2
+            echo 'If you can, please submit a patch.'>&2
+        fi
+    ;;
+    *)
+        printf "Don't know how to list disks for %s operating system.\n" "${uname_s}" >&2
+        printf 'If you can please submit a patch\n' >&2
+    ;;
+esac \
+| xargs

cdist/conf/explorer/hostname

@@ -1,7 +1,6 @@
 #!/bin/sh
 #
-# 2010-2014 Nico Schottelius (nico-cdist at schottelius.org)
-# 2012 Steven Armstrong (steven-cdist at armstrong.cc)
+# 2019 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -19,7 +18,12 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 #
+# Retrieve the running hostname
+#
 
-if command -v uname >/dev/null; then
-   uname -n
+if command -v hostname >/dev/null
+then
+	hostname
+else
+	uname -n
 fi

cdist/conf/explorer/init

@@ -1,7 +1,8 @@
-#!/bin/sh
+#!/bin/sh -e
 #
 # 2016 Daniel Heule (hda at sfs.biz)
 # Copyright 2017, Philippe Gregoire <pg@pgregoire.xyz>
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -19,21 +20,423 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 #
-# Returns the process name of pid 1 ( normaly the init system )
-# for example at linux this value is "init" or "systemd" in most cases
+# Returns the name of the init system (PID 1)
+
+# Expected values:
+# Linux:
+#  Adélie Linux:
+#    sysvinit+openrc
+#  Alpine Linux:
+#    busybox-init+openrc
+#  ArchLinux:
+#    systemd, sysvinit
+#  CRUX:
+#    sysvinit
+#  Debian:
+#    systemd, upstart, sysvinit, openrc, ???
+#  Devuan:
+#    sysvinit, sysvinit+openrc
+#  Gentoo:
+#    sysvinit+openrc, openrc-init, systemd
+#  OpenBMC:
+#    systemd
+#  OpenWrt:
+#    procd, init???
+#  RedHat (RHEL, CentOS, Fedora, RedHat Linux, ...):
+#    systemd, upstart, upstart-legacy, sysvinit
+#  Slackware:
+#    sysvinit
+#  SuSE:
+#    systemd, sysvinit
+#  Ubuntu:
+#    systemd, upstart, upstart-legacy, sysvinit
+#  VoidLinux:
+#    runit
 #
+# GNU:
+#   Debian:
+#     sysvinit, hurd-init
+#
+# BSD:
+#  {Free,Open,Net}BSD:
+#    init
+#
+# Mac OS X:
+#   launchd, init+SystemStarter
+#
+# Solaris/Illumos:
+#   smf, init???
 
-uname_s="$(uname -s)"
+# NOTE: init systems can be stacked. This is popular to run OpenRC on top of
+# sysvinit (Gentoo) or busybox-init (Alpine), but can also be used to run runit
+# as a systemd service.  This makes init system detection very complicated
+# (which result is expected?)  This script tries to untangle some combinations,
+# OpenRC on top of sysv or busybox (X+openrc), but will ignore others (runit as
+# a systemd service)
 
-case "$uname_s" in
-    Linux)
-        (pgrep -P0 -l | awk '/^1[ \t]/ {print $2;}') || true
-    ;;
-    FreeBSD)
-        ps -o comm= -p 1 || true
-    ;;
-    *)
-        # return a empty string as unknown value
-        echo ""
-    ;;
-esac
+# NOTE: When we have no idea, nothing will be printed!
+
+# NOTE:
+# When trying to gather information about the init system make sure to do so
+# without calling the binary!   On some systems this triggers a reinitialisation
+# of the system which we don't want (e.g. embedded systems).
+
+
+set -e
+
+KERNEL_NAME=$(uname -s)
+
+KNOWN_INIT_SYSTEMS=$(cat <<EOF
+systemd
+sysvinit
+upstart
+runit
+procd
+smf
+launchd
+init
+hurd_init
+systemstarter
+EOF
+)
+
+
+common_candidates_by_kernel() {
+	case $KERNEL_NAME
+	in
+		FreeBSD|NetBSD|OpenBSD)
+			echo init
+			;;
+		Linux)
+			echo systemd
+			echo sysvinit
+			echo upstart
+			;;
+		GNU)
+			echo sysvinit
+			echo hurd-init
+			;;
+		Darwin)
+			echo launchd
+			echo systemstarter
+			;;
+		SunOS)
+			echo smf
+			;;
+	esac
+}
+
+
+## Helpers
+
+trim() {
+	sed -e 's/^[[:blank:]]*//' -e 's/[[:blank:]]*$//' -e '/^[[:blank:]]*$/d'
+}
+
+unique() {
+	# Delete duplicate lines (keeping input order)
+	# NOTE: Solaris AWK breaks without if/print construct.
+	awk '{ if (!x[$0]++) print }'
+}
+
+
+## Check functions
+# These functions are used to verify if a guess is correct by checking some
+# common property of a running system (presence of a directory in /run etc.)
+
+check_busybox_init() (
+	busybox_path=${1:-/bin/busybox}
+	test -x "${busybox_path}" || return 1
+	grep -q 'BusyBox v[0-9]' "${busybox_path}" || return 1
+
+	# It is quite common to use Busybox init to stack other init systemd
+	# (like OpenRC) on top of it. So we check for that, too.
+	if stacked=$(check_openrc)
+	then
+		echo "busybox-init+${stacked}"
+	else
+		echo busybox-init
+	fi
+)
+
+check_hurd_init() (
+	init_exe=${1:-/hurd/init}
+	test -x "${init_exe}" || return 1
+	grep -q 'GNU Hurd' "${init_exe}" || return 1
+	echo hurd-init
+)
+
+check_init() {
+	# Checks for various BSD inits...
+	test -x /sbin/init || return 1
+
+	if grep -q -E '(Free|Net|Open)BSD' /sbin/init
+	then
+		echo init
+		return 0
+	fi
+}
+
+check_launchd() {
+	command -v launchctl >/dev/null 2>&1 || return 1
+	launchctl getenv PATH >/dev/null || return 1
+	echo launchd
+}
+
+check_openrc() {
+	test -f /run/openrc/softlevel || return 1
+	echo openrc
+}
+
+check_procd() (
+	procd_path=${1:-/sbin/procd}
+	test -x "${procd_path}" || return 1
+	grep -q 'procd' "${procd_path}" || return 1
+	echo procd
+)
+
+check_runit() {
+	test -d /run/runit || return 1
+	echo runit
+}
+
+check_smf() {
+	# XXX: Is this the correct way??
+	test -f /etc/svc/volatile/svc_nonpersist.db || return 1
+	echo smf
+}
+
+check_systemd() {
+	# NOTE: sd_booted(3)
+	test -d /run/systemd/system/ || return 1
+	# systemctl --version | sed -e '/^systemd/!d;s/^systemd //'
+	echo systemd
+}
+
+check_systemstarter() {
+	test -d /System/Library/StartupItems/ || return 1
+	test -f /System/Library/StartupItems/LoginWindow/StartupParameters.plist || return 1
+	echo init+SystemStarter
+}
+
+check_sysvinit() (
+	init_path=${1:-/sbin/init}
+	test -x "${init_path}" || return 1
+	grep -q 'INIT_VERSION=sysvinit-[0-9.]*' "${init_path}" || return 1
+
+	# It is quite common to use SysVinit to stack other init systemd
+	# (like OpenRC) on top of it. So we check for that, too.
+	if stacked=$(check_openrc)
+	then
+		echo "sysvinit+${stacked}"
+	else
+		echo sysvinit
+	fi
+	unset stacked
+)
+
+check_upstart() {
+	test -x "$(command -v initctl)" || return 1
+	case $(initctl version)
+	in
+		*'(upstart '*')')
+			if test -d /etc/init
+			then
+				# modern (DBus-based?) upstart >= 0.5
+				echo upstart
+			elif test -d /etc/event.d
+			then
+				# ancient upstart
+				echo upstart-legacy
+			else
+				# whatever...
+				echo upstart
+			fi
+			;;
+		*)
+			return 1
+			;;
+	esac
+}
+
+find_init_procfs() (
+	# First, check if the required file in procfs exists...
+	test -h /proc/1/exe || return 1
+
+	# Find init executable
+	init_exe=$(ls -l /proc/1/exe 2>/dev/null) || return 1
+	init_exe=${init_exe#* -> }
+
+	if ! test -x "$init_exe"
+	then
+		# On some rare occasions it can happen that the
+		# running init's binary has been replaced. In this
+		# case Linux adjusts the symlink to "X (deleted)"
+
+		# [root@fedora-12 ~]# readlink /proc/1/exe
+		# /sbin/init (deleted)
+		# [root@fedora-12 ~]# ls -l /proc/1/exe
+		# lrwxrwxrwx. 1 root root 0 2020-01-30 23:00 /proc/1/exe -> /sbin/init (deleted)
+
+		init_exe=${init_exe% (deleted)}
+		test -x "$init_exe" || return 1
+	fi
+
+	echo "${init_exe}"
+)
+
+guess_by_path() {
+	case $1
+	in
+		/bin/busybox)
+			check_busybox_init "$1" && return
+			;;
+		/lib/systemd/systemd)
+			check_systemd "$1" && return
+			;;
+		/hurd/init)
+			check_hurd_init "$1" && return
+			;;
+		/sbin/launchd)
+			check_launchd "$1" && return
+			;;
+		/usr/bin/runit|/sbin/runit)
+			check_runit "$1" && return
+			;;
+		/sbin/openrc-init)
+			if check_openrc "$1" >/dev/null
+			then
+				echo openrc-init
+				return
+			fi
+			;;
+		/sbin/procd)
+			check_procd "$1" && return
+			;;
+		/sbin/init|*/init)
+			# init: it could be anything -> (explicit) no match
+			return 1
+			;;
+	esac
+
+	# No match
+	return 1
+}
+
+guess_by_comm_name() {
+	case $1
+	in
+		busybox)
+			check_busybox_init && return
+			;;
+		openrc-init)
+			if check_openrc >/dev/null
+			then
+				echo openrc-init
+				return 0
+			fi
+			;;
+		init)
+			# init could be anything -> no match
+			return 1
+			;;
+		*)
+			# Run check function by comm name if available.
+			# Fall back to comm name if either it does not exist or
+			# returns non-zero.
+			if type "check_$1" >/dev/null
+			then
+				"check_$1" && return
+			else
+				echo "$1" ; return 0
+			fi
+	esac
+
+	return 1
+}
+
+check_list() (
+	# List must be a multi-line input on stdin (one name per line)
+	while read -r init
+	do
+		"check_${init}" || continue
+		return 0
+	done
+	return 1
+)
+
+
+# BusyBox's versions of ps and pgrep do not support some options
+# depending on which compile-time options have been used.
+
+find_init_pgrep() {
+	pgrep -P0 -fl 2>/dev/null | awk -F '[[:blank:]]' '$1 == 1 { print $2 }'
+}
+
+find_init_ps() {
+	case $KERNEL_NAME
+	in
+		Darwin)
+			ps -o command -p 1 2>/dev/null | tail -n +2
+			;;
+		FreeBSD)
+			ps -o args= -p 1 2>/dev/null | cut -d ' ' -f 1
+			;;
+		Linux)
+			ps -o comm= -p 1 2>/dev/null
+			;;
+		NetBSD)
+			ps -o comm= -p 1 2>/dev/null
+			;;
+		OpenBSD)
+			ps -o args -p 1 2>/dev/null | tail -n +2 | cut -d ' ' -f 1
+			;;
+		*)
+			ps -o args= -p 1 2>/dev/null
+			;;
+	esac | trim  # trim trailing whitespace (some ps like Darwin add it)
+}
+
+find_init() {
+	case $KERNEL_NAME
+	in
+		Linux|GNU|NetBSD)
+			find_init_procfs || find_init_pgrep || find_init_ps
+			;;
+		FreeBSD)
+			find_init_procfs || find_init_ps
+			;;
+		OpenBSD)
+			find_init_pgrep || find_init_ps
+			;;
+		Darwin|SunOS)
+			find_init_ps
+			;;
+		*)
+			echo "Don't know how to determine init." >&2
+			echo 'Please send a patch.' >&2
+			exit 1
+	esac
+}
+
+# -----
+
+init=$(find_init)
+
+# If we got a path, guess by the path first (fall back to file name if no match)
+# else guess by file name directly.
+# shellcheck disable=SC2015
+{
+	test -x "${init}" \
+		&& guess_by_path "${init}" \
+		|| guess_by_comm_name "$(basename "${init}")"
+} && exit 0 || true
+
+
+# Guessing based on the file path and name didn’t lead to a definitive result.
+#
+# We go through all of the checks until we find a match. To speed up the
+# process, common cases will be checked first based on the underlying kernel.
+
+{ common_candidates_by_kernel; echo "${KNOWN_INIT_SYSTEMS}"; } \
+	| unique | check_list

cdist/conf/explorer/interfaces

@@ -1,6 +1,6 @@
-#!/bin/sh
+#!/bin/sh -e
 #
-# 2012 Sébastien Gross <seb•ɑƬ•chezwam•ɖɵʈ•org>
+# 2019 Ander Punnar (ander-at-kvlt-dot-ee)
 #
 # This file is part of cdist.
 #
@@ -17,35 +17,12 @@
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
-#
-# List all network interfaces in explorer/ifaces. One interface per line.
-#
-# If your OS is not supported please provide a ifconfig output
-#
 
-# Use ip, if available
-if command -v ip >/dev/null; then
-    ip -o link show | sed -n 's/^[0-9]\+: \(.\+\): <.*/\1/p'
-    exit 0
-fi
-
-if ! command -v ifconfig >/dev/null; then
-   # no ifconfig, nothing we could do
-   exit 0
-fi
-
-uname_s="$(uname -s)"
-REGEXP='s/^(.*)(:[[:space:]]*flags=|Link encap).*/\1/p'
-
-case "$uname_s" in
-    Darwin)
-	ifconfig -a | sed -n -E "$REGEXP"
-	;;
-    Linux|*BSD)
-	ifconfig -a | sed -n -r "$REGEXP"
-	;;
-    *)
-	echo "Unsupported ifconfig output for $uname_s" >&2
-	exit 1
-	;;
-esac
+if command -v ip >/dev/null
+then
+	ip -o link show | sed -n 's/^[0-9]\+: \(.\+\): <.*/\1/p'
+elif command -v ifconfig >/dev/null
+then
+	ifconfig -a | sed -n -E 's/^(.*)(:[[:space:]]*flags=|Link encap).*/\1/p'
+fi \
+ | sort -u

cdist/conf/explorer/is-freebsd-jail

@@ -0,0 +1,2 @@
+#!/bin/sh
+sysctl -n security.jail.jailed 2>/dev/null | grep "1" || true

cdist/conf/explorer/kernel_name

@@ -1 +1,2 @@
+#!/bin/sh
 uname -s

cdist/conf/explorer/lsb_codename

@@ -20,8 +20,9 @@
 #
 
 set +e
-case "$($__explorer/os)" in
+case "$("$__explorer/os")" in
    openwrt)
+      # shellcheck disable=SC1091
       (. /etc/openwrt_release && echo "$DISTRIB_CODENAME")
    ;;
    *)

cdist/conf/explorer/lsb_description

@@ -20,8 +20,9 @@
 #
 
 set +e
-case "$($__explorer/os)" in
+case "$("$__explorer/os")" in
    openwrt)
+      # shellcheck disable=SC1091
       (. /etc/openwrt_release && echo "$DISTRIB_DESCRIPTION")
    ;;
    *)

cdist/conf/explorer/lsb_id

@@ -20,8 +20,9 @@
 #
 
 set +e
-case "$($__explorer/os)" in
+case "$("$__explorer/os")" in
    openwrt)
+      # shellcheck disable=SC1091
       (. /etc/openwrt_release && echo "$DISTRIB_ID")
    ;;
    *)

cdist/conf/explorer/lsb_release

@@ -20,8 +20,9 @@
 #
 
 set +e
-case "$($__explorer/os)" in
+case "$("$__explorer/os")" in
    openwrt)
+      # shellcheck disable=SC1091
       (. /etc/openwrt_release && echo "$DISTRIB_RELEASE")
    ;;
    *)

cdist/conf/explorer/machine

@@ -22,6 +22,6 @@
 #
 #
 
-if command -v uname 2>&1 >/dev/null; then
+if command -v uname >/dev/null 2>&1 ; then
    uname -m
 fi

cdist/conf/explorer/machine_type

@@ -2,6 +2,7 @@
 #
 # 2014 Daniel Heule  (hda at sfs.biz)
 # 2014 Thomas Oettli (otho at sfs.biz)
+# 2020 Evilham (contact at evilham.com)
 #
 # This file is part of cdist.
 #
@@ -18,63 +19,91 @@
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
-#
 
-# FIXME: other system types (not linux ...)
+os=$("$__explorer/os")
 
-if [ -d "/proc/vz" -a ! -d "/proc/bc" ]; then
-    echo openvz
-    exit
-fi
-
-if [ -e "/proc/1/environ" ] &&
-    cat "/proc/1/environ" | tr '\000' '\n' | grep -Eiq '^container='; then
-    echo lxc
-    exit
-fi
-
-if [ -r /proc/cpuinfo ]; then
-    # this should only exist on virtual guest machines,
-    # tested on vmware, xen, kvm
-    if grep -q "hypervisor" /proc/cpuinfo; then
-        # this file is aviable in xen guest systems
-        if [ -r /sys/hypervisor/type ]; then
-            if grep -q -i "xen" /sys/hypervisor/type; then
-                echo virtual_by_xen
-                exit 
+vendor_string_to_machine_type() {
+    for vendor in vmware bochs kvm qemu virtualbox bhyve; do
+        if echo "${1}" | grep -q -i "${vendor}"; then
+            if [ "${vendor}" = "bochs" ] || [ "${vendor}" = "qemu" ]; then
+                vendor="kvm"
             fi
-        else
-            if [ -r /sys/class/dmi/id/product_name ]; then
-                if grep -q -i 'vmware' /sys/class/dmi/id/product_name; then
-                    echo "virtual_by_vmware"
-                    exit
-                elif grep -q -i 'bochs' /sys/class/dmi/id/product_name; then
-                    echo "virtual_by_kvm"
-                    exit 
-                elif grep -q -i 'virtualbox' /sys/class/dmi/id/product_name; then
-                    echo "virtual_by_virtualbox"
-                    exit 
-                fi
-            fi
-
-            if [ -r /sys/class/dmi/id/sys_vendor ]; then
-                if grep -q -i 'qemu' /sys/class/dmi/id/sys_vendor; then
-                    echo "virtual_by_kvm"
-                    exit
-                fi
-            fi
-
-            if [ -r /sys/class/dmi/id/chassis_vendor ]; then
-                if grep -q -i 'qemu' /sys/class/dmi/id/chassis_vendor; then
-                    echo "virtual_by_kvm"
-                    exit
-                fi
-            fi                        
+            echo "virtual_by_${vendor}"
+            exit
         fi
-        echo "virtual_by_unknown"
-    else
-        echo "physical"
-    fi
-else
-    echo "unknown"
-fi
+    done
+}
+
+case "$os" in
+    "freebsd")
+        # FreeBSD does not have /proc/cpuinfo even when procfs is used.
+        # Instead there is a sysctl kern.vm_guest.
+        # Which is 'none' if physical, else the virtualisation.
+        vm_guest="$(sysctl -n kern.vm_guest 2>/dev/null || true)"
+        if [ -n "${vm_guest}" ]; then
+            if [ "${vm_guest}" = "none" ]; then
+                echo "physical"
+                exit
+            fi
+            echo "virtual_by_${vm_guest}"
+            exit
+        fi
+    ;;
+
+    "openbsd")
+        # OpenBSD can also use the sysctl's: hw.vendor or hw.product.
+        # Note we can be reasonably sure about a machine being virtualised
+        # as long as we can identify the virtualisation technology.
+        # But not so much about it being physical...
+        # Patches are welcome / reach out if you have better ideas.
+        for sysctl in hw.vendor hw.product; do
+            # This exits if we can make a reasonable judgement
+            vendor_string_to_machine_type "$(sysctl -n "${sysctl}")"
+        done
+    ;;
+
+    *)
+        # Defaulting to linux for compatibility with previous cdist behaviour
+
+        if [ -d "/proc/vz" ] && [ ! -d "/proc/bc" ]; then
+            echo openvz
+            exit
+        fi
+
+        if [ -e "/proc/1/environ" ] &&
+            tr '\000' '\n' < "/proc/1/environ" | grep -Eiq '^container='; then
+            echo lxc
+            exit
+        fi
+
+        if [ -r /proc/cpuinfo ]; then
+            # this should only exist on virtual guest machines,
+            # tested on vmware, xen, kvm, bhyve
+            if grep -q "hypervisor" /proc/cpuinfo; then
+                # this file is aviable in xen guest systems
+                if [ -r /sys/hypervisor/type ]; then
+                    if grep -q -i "xen" /sys/hypervisor/type; then
+                        echo virtual_by_xen
+                        exit
+                    fi
+                else
+                    for vendor_file in /sys/class/dmi/id/product_name \
+                                       /sys/class/dmi/id/sys_vendor \
+                                       /sys/class/dmi/id/chasis_vendor; do
+                        if [ -r ${vendor_file} ]; then
+                            # This exits if we can make a reasonable judgement
+                            vendor_string_to_machine_type "$(cat "${vendor_file}")"
+                        fi
+                    done
+                fi
+                echo "virtual_by_unknown"
+                exit
+            else
+                echo "physical"
+                exit
+            fi
+        fi
+    ;;
+esac
+
+echo "unknown"

cdist/conf/explorer/memory

@@ -2,6 +2,7 @@
 #
 # 2014 Daniel Heule  (hda at sfs.biz)
 # 2014 Thomas Oettli (otho at sfs.biz)
+# Copyright 2017, Philippe Gregoire <pg@pgregoire.xyz>
 #
 # This file is part of cdist.
 #
@@ -28,6 +29,11 @@ case "$os" in
         echo "$(sysctl -n hw.memsize)/1024" | bc
     ;;
 
+    *"bsd")
+        PATH=$(getconf PATH)
+        echo "$(sysctl -n hw.physmem) / 1048576" | bc
+    ;;
+
     *)
     if [ -r /proc/meminfo ]; then
         grep "MemTotal:" /proc/meminfo | awk '{print $2}'

cdist/conf/explorer/os

@@ -51,15 +51,17 @@ if grep -q ^DISTRIB_ID=Ubuntu /etc/lsb-release 2>/dev/null; then
    exit 0
 fi
 
+# devuan ascii has both devuan_version and debian_version, so we need to check devuan_version first!
+if [ -f /etc/devuan_version ]; then
+   echo devuan
+   exit 0
+fi
+
 if [ -f /etc/debian_version ]; then
    echo debian
    exit 0
 fi
 
-if [ -f /etc/devuan_version ]; then
-   echo devuan
-   exit 0
-fi
 ###
 
 if [ -f /etc/gentoo-release ]; then
@@ -141,9 +143,16 @@ case "$uname_s" in
 esac
 
 if [ -f /etc/os-release ]; then
+   # after sles15, suse don't provide an /etc/SuSE-release anymore, but there is almost no difference between sles and opensuse leap, so call it suse
+   # shellcheck disable=SC1091
+   if (. /etc/os-release && echo "${ID_LIKE}" | grep -q '\(^\|\ \)suse\($\|\ \)')
+   then
+      echo suse
+      exit 0
+   fi
    # already lowercase, according to:
    # https://www.freedesktop.org/software/systemd/man/os-release.html
-   awk -F= '/^ID=/ {print $2;}' /etc/os-release
+   awk -F= '/^ID=/ { if ($2 ~ /^'"'"'(.*)'"'"'$/ || $2 ~ /^"(.*)"$/) { print substr($2, 2, length($2) - 2) } else { print $2 } }' /etc/os-release
    exit 0
 fi
 

cdist/conf/explorer/os_release

@@ -1,6 +1,7 @@
 #!/bin/sh
 #
-# 2014 Nico Schottelius (nico-cdist at schottelius.org)
+# 2018 Adam Dej (dejko.a at gmail.com)
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -18,13 +19,20 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 #
-# Retrieve the contents of /etc/hostname
-#
 
-# Almost any distribution
-if [ -f /etc/hostname ]; then
-    cat /etc/hostname
-# SuSE
-elif [ -f /etc/HOSTNAME ]; then
-    cat /etc/HOSTNAME
+# See os-release(5) and http://0pointer.de/blog/projects/os-release
+
+if test -f /etc/os-release
+then
+	# Linux and FreeBSD (usually a symlink)
+	cat /etc/os-release
+elif test -f /usr/lib/os-release
+then
+	# systemd
+	cat /usr/lib/os-release
+elif test -f /var/run/os-release
+then
+	# FreeBSD (created by os-release service)
+	cat /var/run/os-release
 fi
+

cdist/conf/explorer/os_version

@@ -22,7 +22,7 @@
 #
 #
 
-case "$($__explorer/os)" in
+case "$("$__explorer/os")" in
    amazon)
       cat /etc/system-release
    ;;
@@ -31,7 +31,32 @@ case "$($__explorer/os)" in
       cat /etc/arch-release
    ;;
    debian)
-      cat /etc/debian_version
+      debian_version=$(cat /etc/debian_version)
+      case $debian_version
+      in
+          testing/unstable)
+              # previous to Debian 4.0 testing/unstable was used
+              # cf. https://metadata.ftp-master.debian.org/changelogs/main/b/base-files/base-files_11_changelog
+              echo 3.99
+              ;;
+          */sid)
+              # sid versions don't have a number, so we decode by codename:
+              case $(expr "$debian_version" : '\([a-z]\{1,\}\)/')
+              in
+                  bullseye) echo 10.99 ;;
+                  buster) echo 9.99 ;;
+                  stretch) echo 8.99 ;;
+                  jessie) echo 7.99 ;;
+                  wheezy) echo 6.99 ;;
+                  squeeze) echo 5.99 ;;
+                  lenny) echo 4.99 ;;
+                  *) exit 1
+              esac
+              ;;
+          *)
+              echo "$debian_version"
+              ;;
+      esac
    ;;
    devuan)
       cat /etc/devuan_version
@@ -45,6 +70,11 @@ case "$($__explorer/os)" in
    macosx)
       sw_vers -productVersion
    ;;
+   freebsd)
+      # Apparently uname -r is not a reliable way to get the patch level.
+      # See: https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=251743
+      freebsd-version
+   ;;
    *bsd|solaris)
       uname -r
    ;;
@@ -70,4 +100,7 @@ case "$($__explorer/os)" in
    ubuntu)
       lsb_release -sr
    ;;
+   alpine)
+       cat /etc/alpine-release
+   ;;
 esac

cdist/conf/type/__acl/explorer/acl_is

@@ -0,0 +1,31 @@
+#!/bin/sh -e
+#
+# 2018 Ander Punnar (ander-at-kvlt-dot-ee)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+[ ! -e "/$__object_id" ] && exit 0
+
+if ! command -v getfacl > /dev/null
+then
+    echo 'getfacl not available' >&2
+    exit 1
+fi
+
+getfacl "/$__object_id" 2>/dev/null \
+    | grep -Eo '^(default:)?(user|group|(mask|other):):[^:][[:graph:]]+' \
+    || true

cdist/conf/type/__acl/explorer/file_is

@@ -0,0 +1,31 @@
+#!/bin/sh -e
+#
+# 2018 Ander Punnar (ander-at-kvlt-dot-ee)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+if [ -e "/$__object_id" ]
+then
+    if [ -d "/$__object_id" ]
+    then echo directory
+    elif [ -f "/$__object_id" ]
+    then echo regular
+    else echo other
+    fi
+else
+    echo missing
+fi

cdist/conf/type/__acl/explorer/getent

@@ -0,0 +1,4 @@
+#!/bin/sh -e
+
+getent passwd | awk -F: '{print "user:"$1}'
+getent group | awk -F: '{print "group:"$1}'

cdist/conf/type/__acl/gencode-remote

@@ -0,0 +1,143 @@
+#!/bin/sh -e
+#
+# 2018 Ander Punnar (ander-at-kvlt-dot-ee)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+file_is="$( cat "$__object/explorer/file_is" )"
+
+if [ "$file_is" = 'missing' ] \
+    && [ -z "$__cdist_dry_run" ] \
+    && [ ! -f "$__object/parameter/file" ] \
+    && [ ! -f "$__object/parameter/directory" ]
+then
+    exit 0
+fi
+
+os="$( cat "$__global/explorer/os" )"
+
+acl_path="/$__object_id"
+
+acl_is="$( cat "$__object/explorer/acl_is" )"
+
+if [ -f "$__object/parameter/source" ]
+then
+    acl_source="$( cat "$__object/parameter/source" )"
+
+    if [ "$acl_source" = '-' ]
+    then
+        acl_should="$( cat "$__object/stdin" )"
+    else
+        acl_should="$( grep -Ev '^#|^$' "$acl_source" )"
+    fi
+elif [ -f "$__object/parameter/entry" ]
+then
+    acl_should="$( cat "$__object/parameter/entry" )"
+else
+    echo 'no parameters set' >&2
+    exit 1
+fi
+
+# instead of setfacl's non-helpful message "Option -m: Invalid argument near character X"
+# let's check if target has necessary users and groups, since mistyped or missing
+# users/groups in target is most common reason.
+echo "$acl_should" \
+    | grep -Po '(user|group):[^:]+' \
+    | sort -u \
+    | while read -r l
+    do
+        if ! grep "$l" -Fxq "$__object/explorer/getent"
+        then
+            echo "no $l' in target" | sed "s/:/ '/" >&2
+            exit 1
+        fi
+    done
+
+if [ -f "$__object/parameter/default" ]
+then
+    acl_should="$( echo "$acl_should" \
+        | sed 's/^default://' \
+        | sort -u \
+        | sed 's/\(.*\)/default:\1\n\1/' )"
+fi
+
+if [ "$file_is" = 'regular' ] \
+    && echo "$acl_should" | grep -Eq '^default:'
+then
+    # only directories can have default ACLs,
+    # but instead of error,
+    # let's just remove default entries
+    acl_should="$( echo "$acl_should" | grep -Ev '^default:' )"
+fi
+
+if echo "$acl_should" | awk -F: '{ print $NF }' | grep -Fq 'X'
+then
+    [ "$file_is" = 'directory' ] && rep=x || rep=-
+
+    acl_should="$( echo "$acl_should" | sed "s/\\(.*\\)X/\\1$rep/" )"
+fi
+
+setfacl_exec='setfacl'
+
+if [ -f "$__object/parameter/recursive" ]
+then
+    if echo "$os" | grep -Fq 'freebsd'
+    then
+        echo "$os setfacl do not support recursive operations" >&2
+    else
+        setfacl_exec="$setfacl_exec -R"
+    fi
+fi
+
+if [ -f "$__object/parameter/remove" ]
+then
+    echo "$acl_is" | while read -r acl
+    do
+        # skip wanted ACL entries which already exist
+        # and skip mask and other entries, because we
+        # can't actually remove them, but only change.
+        if echo "$acl_should" | grep -Eq "^$acl" \
+            || echo "$acl" | grep -Eq '^(default:)?(mask|other)'
+        then continue
+        fi
+
+        if echo "$os" | grep -Fq 'freebsd'
+        then
+            remove="$acl"
+        else
+            remove="$( echo "$acl" | sed 's/:...$//' )"
+        fi
+
+        echo "$setfacl_exec -x \"$remove\" \"$acl_path\""
+        echo "removed '$remove'" >> "$__messages_out"
+    done
+fi
+
+for acl in $acl_should
+do
+    if ! echo "$acl_is" | grep -Eq "^$acl"
+    then
+        if echo "$os" | grep -Fq 'freebsd' \
+            && echo "$acl" | grep -Eq '^default:'
+        then
+            echo "setting default ACL in $os is currently not supported" >&2
+        else
+            echo "$setfacl_exec -m \"$acl\" \"$acl_path\""
+            echo "added '$acl'" >> "$__messages_out"
+        fi
+    fi
+done

cdist/conf/type/__acl/man.rst

@@ -0,0 +1,108 @@
+cdist-type__acl(7)
+==================
+
+NAME
+----
+cdist-type__acl - Set ACL entries
+
+
+DESCRIPTION
+-----------
+Fully supported and tested on Linux (ext4 filesystem), partial support for FreeBSD.
+
+See ``setfacl`` and ``acl`` manpages for more details.
+
+One of ``--entry`` or ``--source`` must be used.
+
+
+OPTIONAL MULTIPLE PARAMETERS
+----------------------------
+entry
+   Set ACL entry following ``getfacl`` output syntax.
+   Must be used if ``--source`` is not used.
+
+
+OPTIONAL PARAMETERS
+-------------------
+source
+   Read ACL entries from stdin or file.
+   Ordering of entries is not important.
+   When reading from file, comments and empty lines are ignored.
+   Must be used if ``--entry`` is not used.
+
+file
+   Create/change file with ``__file`` using ``user:group:mode`` pattern.
+
+directory
+   Create/change directory with ``__directory`` using ``user:group:mode`` pattern.
+
+
+BOOLEAN PARAMETERS
+------------------
+default
+   Set all ACL entries as default too.
+   Only directories can have default ACLs.
+   Setting default ACL in FreeBSD is currently not supported.
+
+recursive
+   Make ``setfacl`` recursive (Linux only), but not ``getfacl`` in explorer.
+
+remove
+   Remove undefined ACL entries.
+   ``mask`` and ``other`` entries can't be removed, but only changed.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    __acl /srv/project \
+        --default \
+        --recursive \
+        --remove \
+        --entry user:alice:rwx \
+        --entry user:bob:r-x \
+        --entry group:project-group:rwx \
+        --entry group:some-other-group:r-x \
+        --entry mask::r-x \
+        --entry other::r-x
+
+    # give Alice read-only access to subdir,
+    # but don't allow her to see parent content.
+
+    __acl /srv/project2 \
+        --remove \
+        --entry default:group:secret-project:rwx \
+        --entry group:secret-project:rwx \
+        --entry user:alice:--x
+
+    __acl /srv/project2/subdir \
+        --default \
+        --remove \
+        --entry group:secret-project:rwx \
+        --entry user:alice:r-x
+
+    # read acl from stdin
+    echo 'user:alice:rwx' \
+        | __acl /path/to/directory --source -
+
+    # create/change directory too
+    __acl /path/to/directory \
+        --default \
+        --remove \
+        --directory root:root:770 \
+        --entry user:nobody:rwx
+
+
+AUTHORS
+-------
+Ander Punnar <ander-at-kvlt-dot-ee>
+
+
+COPYING
+-------
+Copyright \(C) 2018 Ander Punnar. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.

cdist/conf/type/__acl/manifest

@@ -0,0 +1,11 @@
+#!/bin/sh -e
+
+for p in file directory
+do
+    [ ! -f "$__object/parameter/$p" ] && continue
+
+    "__$p" "/$__object_id" \
+        --owner "$( awk -F: '{print $1}' "$__object/parameter/$p" )" \
+        --group "$( awk -F: '{print $2}' "$__object/parameter/$p" )" \
+        --mode  "$( awk -F: '{print $3}' "$__object/parameter/$p" )"
+done

cdist/conf/type/__acl/parameter/boolean

@@ -0,0 +1,3 @@
+recursive
+default
+remove

cdist/conf/type/__acl/parameter/optional

@@ -0,0 +1,3 @@
+source
+file
+directory

cdist/conf/type/__acl/parameter/optional_multiple

@@ -0,0 +1 @@
+entry

cdist/conf/type/__apt_backports/man.rst

@@ -0,0 +1,104 @@
+cdist-type__debian_backports(7)
+===============================
+
+NAME
+----
+cdist-type__apt_backports - Install backports
+
+
+DESCRIPTION
+-----------
+This singleton type installs backports for the current OS release.
+It aborts if backports are not supported for the specified OS or
+no version codename could be fetched (like Debian unstable).
+
+The package index will be automatically updated if required.
+
+It supports backports from following OSes:
+
+- Debian
+- Devuan
+- Ubuntu
+
+
+REQUIRED PARAMETERS
+-------------------
+None.
+
+
+OPTIONAL PARAMETERS
+-------------------
+state
+    Represents the state of the backports repository. ``present`` or
+    ``absent``, defaults to ``present``.
+
+    Will be directly passed to :strong:`cdist-type__apt_source`\ (7).
+
+mirror
+    The mirror to fetch the backports from. Will defaults to the generic
+    mirror of the current OS.
+
+    Will be directly passed to :strong:`cdist-type__apt_source`\ (7).
+
+
+BOOLEAN PARAMETERS
+------------------
+None.
+
+
+MESSAGES
+--------
+None.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+   # setup the backports
+   __apt_backports
+   __apt_backports --state absent
+   __apt_backports --state present --mirror "http://ftp.de.debian.org/debian/"
+
+   # install a backports package
+   # currently for the buster release backports
+   require="__apt_backports" __package_apt wireguard \
+        --target-release buster-backports
+
+
+ABORTS
+------
+Aborts if the detected os is not Debian.
+
+Aborts if no distribuition codename could be detected. This is common for the
+unstable distribution, but there is no backports repository for it already.
+
+
+CAVEATS
+-------
+For Ubuntu, it setup all componenents for the backports repository: ``main``,
+``restricted``, ``universe`` and ``multiverse``. The user may not want to
+install proprietary packages, which will only be installed if the user
+explicitly uses the backports target-release. The user may change this behavior
+to install backports packages without the need of explicitly select it.
+
+
+SEE ALSO
+--------
+`Official Debian Backports site <https://backports.debian.org/>`_
+
+:strong:`cdist-type__apt_source`\ (7)
+
+
+AUTHORS
+-------
+Matthias Stecher <matthiasstecher at gmx.de>
+
+
+COPYING
+-------
+Copyright \(C) 2020 Matthias Stecher. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.

cdist/conf/type/__apt_backports/manifest

@@ -0,0 +1,81 @@
+#!/bin/sh -e
+# __apt_backports/manifest
+#
+# 2020 Matthias Stecher (matthiasstecher at gmx.de)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+# Enables/disables backports repository. Utilises __apt_source for it.
+#
+
+
+# Get the distribution codename by /etc/os-release.
+#  is already executed in a subshell by string substitution
+#  lsb_release may not be given in all installations
+codename_os_release() {
+    # shellcheck disable=SC1090
+    . "$__global/explorer/os_release"
+    printf "%s" "$VERSION_CODENAME"
+}
+
+# detect backport distribution
+os="$(cat "$__global/explorer/os")"
+case "$os" in
+    debian)
+        dist="$( codename_os_release )"
+        components="main"
+        mirror="http://deb.debian.org/debian/"
+        ;;
+    devuan)
+        dist="$( codename_os_release )"
+        components="main"
+        mirror="http://deb.devuan.org/merged"
+        ;;
+    ubuntu)
+        dist="$( codename_os_release )"
+        components="main restricted universe multiverse"
+        mirror="http://archive.ubuntu.com/ubuntu"
+        ;;
+
+    *)
+        printf "Backports for %s are not supported!\n" "$os" >&2
+        exit 1
+        ;;
+esac
+
+# error if no codename given (e.g. on Debian unstable)
+if [ -z "$dist" ]; then
+    printf "No backports for unkown version of distribution %s!\n" "$os" >&2
+    exit 1
+fi
+
+
+# parameters
+state="$(cat "$__object/parameter/state")"
+
+# mirror already set for the os, only override user-values
+if [ -f "$__object/parameter/mirror" ]; then
+    mirror="$(cat "$__object/parameter/mirror")"
+fi
+
+
+# install the given backports repository
+__apt_source "${dist}-backports" \
+    --state "$state" \
+    --distribution "${dist}-backports" \
+    --component "$components" \
+    --uri "$mirror"

cdist/conf/type/__apt_backports/parameter/default/state

@@ -0,0 +1 @@
+present

cdist/conf/type/__apt_backports/parameter/optional

@@ -0,0 +1,2 @@
+state
+mirror

cdist/conf/type/__apt_default_release/man.rst

@@ -0,0 +1,46 @@
+cdist-type__apt_default_release(7)
+==================================
+
+NAME
+----
+cdist-type__apt_default_release - Configure the default release for apt
+
+
+DESCRIPTION
+-----------
+Configure the default release for apt, using the APT::Default-Release
+configuration value.
+
+REQUIRED PARAMETERS
+-------------------
+release
+   The value to set APT::Default-Release to.
+
+   This can contain release name, codename or release version. Examples:
+   'stable', 'testing', 'unstable', 'stretch', 'buster', '4.0', '5.0*'.
+
+
+OPTIONAL PARAMETERS
+-------------------
+None.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    __apt_default_release --release stretch
+
+
+AUTHORS
+-------
+Matthijs Kooijman <matthijs--@--stdin.nl>
+
+
+COPYING
+-------
+Copyright \(C) 2017 Matthijs Kooijman. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.

cdist/conf/type/__apt_default_release/manifest

@@ -1,6 +1,7 @@
-#!/bin/sh
+#!/bin/sh -e
 #
-# 2012 Jake Guffey (jake.guffey at eprotex.com)
+# 2014 Steven Armstrong (steven-cdist at armstrong.cc)
+# 2017 Matthijs Kooijman (matthijs at stdin.nl)
 #
 # This file is part of cdist.
 #
@@ -17,20 +18,24 @@
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
-#
-# Get the location of the pf ruleset on the target host.
-#
 
-# Debug
-#exec >&2
-#set -x
 
-# Check /etc/rc.conf for pf's configuration file name. Default to /etc/pf.conf
-
-RC="/etc/rc.conf"
-PFCONF="$(grep '^pf_rules=' ${RC} | cut -d= -f2 | sed 's/"//g')"
-echo ${PFCONF:-"/etc/pf.conf"}
-
-# Debug
-#set +x
+os=$(cat "$__global/explorer/os")
+release="$(cat "$__object/parameter/release")"
 
+case "$os" in
+   ubuntu|debian|devuan)
+      __file /etc/apt/apt.conf.d/99-default-release \
+         --owner root --group root --mode 644 \
+         --source - << DONE
+APT::Default-Release "$release";
+DONE
+   ;;
+   *)
+      cat >&2 << DONE
+The developer of this type (${__type##*/}) did not think your operating system
+($os) would have any use for it. If you think otherwise please submit a patch.
+DONE
+      exit 1
+   ;;
+esac

cdist/conf/type/__apt_default_release/parameter/required

@@ -0,0 +1 @@
+release

cdist/conf/type/__apt_key/explorer/state

@@ -27,6 +27,18 @@ else
    keyid="$__object_id"
 fi
 
-apt-key export "$keyid" | head -n 1 | grep -Fqe "BEGIN PGP PUBLIC KEY BLOCK" \
-   && echo present \
-   || echo absent
+keydir="$(cat "$__object/parameter/keydir")"
+keyfile="$keydir/$__object_id.gpg"
+
+if [ -d "$keydir" ]
+then
+   if [ -f "$keyfile" ]
+   then echo present
+   else echo absent
+   fi
+else
+   # fallback to deprecated apt-key
+   apt-key export "$keyid" | head -n 1 | grep -Fqe "BEGIN PGP PUBLIC KEY BLOCK" \
+      && echo present \
+      || echo absent
+fi

cdist/conf/type/__apt_key/gencode-remote

@@ -31,12 +31,82 @@ if [ "$state_should" = "$state_is" ]; then
    exit 0
 fi
 
+keydir="$(cat "$__object/parameter/keydir")"
+keyfile="$keydir/$__object_id.gpg"
+
 case "$state_should" in
    present)
       keyserver="$(cat "$__object/parameter/keyserver")"
-      echo "apt-key adv --keyserver \"$keyserver\" --recv-keys \"$keyid\""
+
+      if [ -f "$__object/parameter/uri" ]; then
+         uri="$(cat "$__object/parameter/uri")"
+
+         if [ -d "$keydir" ]; then
+            cat << EOF
+
+curl -s -L \\
+    -o "$keyfile" \\
+    "$uri"
+
+key="\$( cat "$keyfile" )"
+
+if echo "\$key" | grep -Fq 'BEGIN PGP PUBLIC KEY BLOCK'
+then
+    echo "\$key" | gpg --dearmor > "$keyfile"
+fi
+
+EOF
+         else
+            # fallback to deprecated apt-key
+            echo "curl -s -L '$uri' | apt-key add -"
+         fi
+      elif [ -d "$keydir" ]; then
+         # we need to kill gpg after 30 seconds, because gpg
+         # can get stuck if keyserver is not responding.
+         # exporting env var and not exit 1,
+         # because we need to clean up and kill dirmngr.
+         cat << EOF
+
+gpgtmphome="\$( mktemp -d )"
+
+if timeout 30s \\
+    gpg --homedir "\$gpgtmphome" \\
+        --keyserver "$keyserver" \\
+        --recv-keys "$keyid"
+then
+    gpg --homedir "\$gpgtmphome" \\
+        --export "$keyid" \\
+        > "$keyfile"
+else
+    export GPG_GOT_STUCK=1
+fi
+
+GNUPGHOME="\$gpgtmphome" gpgconf --kill dirmngr
+
+rm -rf "\$gpgtmphome"
+
+if [ -n "\$GPG_GOT_STUCK" ]
+then
+    echo "GPG GOT STUCK - no response from keyserver after 30 seconds" >&2
+    exit 1
+fi
+
+EOF
+      else
+         # fallback to deprecated apt-key
+         echo "apt-key adv --keyserver \"$keyserver\" --recv-keys \"$keyid\""
+      fi
+
+      echo "added '$keyid'" >> "$__messages_out"
    ;;
    absent)
-      echo "apt-key del \"$keyid\""
+      if [ -f "$keyfile" ]; then
+         echo "rm '$keyfile'"
+      else
+         # fallback to deprecated apt-key
+         echo "apt-key del \"$keyid\""
+      fi
+
+      echo "removed '$keyid'" >> "$__messages_out"
    ;;
 esac

cdist/conf/type/__apt_key/man.rst

@@ -28,6 +28,12 @@ keyserver
    the keyserver from which to fetch the key. If omitted the default set
    in ./parameter/default/keyserver is used.
 
+keydir
+   key save location, defaults to ``/etc/apt/trusted.pgp.d``
+
+uri
+   the URI from which to download the key
+
 
 EXAMPLES
 --------
@@ -47,15 +53,20 @@ EXAMPLES
     # same thing with other keyserver
     __apt_key UbuntuArchiveKey --keyid 437D05B5 --keyserver keyserver.ubuntu.com
 
+    # download key from the internet
+    __apt_key rabbitmq \
+       --uri http://www.rabbitmq.com/rabbitmq-signing-key-public.asc
+
 
 AUTHORS
 -------
 Steven Armstrong <steven-cdist--@--armstrong.cc>
+Ander Punnar <ander-at-kvlt-dot-ee>
 
 
 COPYING
 -------
-Copyright \(C) 2011-2014 Steven Armstrong. You can redistribute it
-and/or modify it under the terms of the GNU General Public License as
-published by the Free Software Foundation, either version 3 of the
+Copyright \(C) 2011-2019 Steven Armstrong and Ander Punnar. You can
+redistribute it and/or modify it under the terms of the GNU General Public
+License as published by the Free Software Foundation, either version 3 of the
 License, or (at your option) any later version.

cdist/conf/type/__apt_key/manifest

@@ -0,0 +1,8 @@
+#!/bin/sh -e
+
+__package gnupg
+
+if [ -f "$__object/parameter/uri" ]
+then __package curl
+else __package dirmngr
+fi

cdist/conf/type/__apt_key/parameter/default/keydir

@@ -0,0 +1 @@
+/etc/apt/trusted.gpg.d

cdist/conf/type/__apt_key/parameter/optional

@@ -1,3 +1,5 @@
 state
 keyid
 keyserver
+keydir
+uri

cdist/conf/type/__apt_norecommends/man.rst

@@ -32,11 +32,12 @@ EXAMPLES
 AUTHORS
 -------
 Steven Armstrong <steven-cdist--@--armstrong.cc>
+Dennis Camera <dennis.camera--@--ssrq-sds-fds.ch>
 
 
 COPYING
 -------
-Copyright \(C) 2014 Steven Armstrong. You can redistribute it
-and/or modify it under the terms of the GNU General Public License as
-published by the Free Software Foundation, either version 3 of the
-License, or (at your option) any later version.
+Copyright \(C) 2014 Steven Armstrong, 2020 Dennis Camera.
+You can redistribute it and/or modify it under the terms of the GNU General
+Public License as published by the Free Software Foundation, either version 3 of
+the License, or (at your option) any later version.

cdist/conf/type/__apt_norecommends/manifest

@@ -1,6 +1,7 @@
 #!/bin/sh -e
 #
 # 2014 Steven Armstrong (steven-cdist at armstrong.cc)
+# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
 #
 # This file is part of cdist.
 #
@@ -19,26 +20,28 @@
 #
 
 
-os=$(cat "$__global/explorer/os")
+os=$(cat "${__global:?}/explorer/os")
 
-case "$os" in
-   ubuntu|debian|devuan)
-      # No stinking recommends thank you very much.
-      # If I want something installed I will do so myself.
-      __file /etc/apt/apt.conf.d/99-no-recommends \
-         --owner root --group root --mode 644 \
-         --source - << DONE
-APT::Install-Recommends "0";
-APT::Install-Suggests "0";
-APT::AutoRemove::RecommendsImportant "0";
-APT::AutoRemove::SuggestsImportant "0";
-DONE
-   ;;
-   *)
-      cat >&2 << DONE
+case ${os}
+in
+	(ubuntu|debian|devuan)
+		__file /etc/apt/apt.conf.d/00InstallRecommends --state present \
+			--owner root --group root --mode 0644 --source - <<-'EOF'
+		APT::Install-Recommends "false";
+		APT::Install-Suggests "false";
+		APT::AutoRemove::RecommendsImportant "false";
+		APT::AutoRemove::SuggestsImportant "false";
+		EOF
+
+		# TODO: Remove the following object after some time
+		require=__file/etc/apt/apt.conf.d/00InstallRecommends \
+		__file /etc/apt/apt.conf.d/99-no-recommends --state absent
+		;;
+	(*)
+		cat >&2 <<EOF
 The developer of this type (${__type##*/}) did not think your operating system
 ($os) would have any use for it. If you think otherwise please submit a patch.
-DONE
-      exit 1
-   ;;
+EOF
+		exit 1
+		;;
 esac

cdist/conf/type/__apt_ppa/explorer/state

@@ -23,10 +23,11 @@
 
 name="$__object_id"
 
+# shellcheck disable=SC1091
 . /etc/lsb-release
 
 repo_name="${name#ppa:}"
-repo_file_name="$(echo "$repo_name" | sed -e "s|[/:]|-|" -e "s|\.|_|")-${DISTRIB_CODENAME}.list"
+repo_file_name="$(echo "$repo_name" | sed -e 's|[/:]|-|' -e 's|\.|_|')-${DISTRIB_CODENAME}.list"
 
 [ -s "/etc/apt/sources.list.d/${repo_file_name}" ] \
    && echo present || echo absent

cdist/conf/type/__apt_ppa/gencode-remote

@@ -29,9 +29,9 @@ fi
 
 case "$state_should" in
    present)
-      echo add-apt-repository \"$name\"
+      echo "add-apt-repository '$name'"
    ;;
    absent)
-      echo remove-apt-repository \"$name\"
+      echo "remove-apt-repository '$name'"
    ;;
 esac

cdist/conf/type/__apt_ppa/manifest

@@ -18,8 +18,6 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-name="$__object_id"
-
 __package software-properties-common
 
 require="__package/software-properties-common" \

cdist/conf/type/__apt_source/gencode-remote

@@ -0,0 +1,28 @@
+#!/bin/sh -e
+#
+# 2018 Steven Armstrong (steven-cdist at armstrong.cc)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+name="$__object_id"
+destination="/etc/apt/sources.list.d/${name}.list"
+
+if grep -q "^__file${destination}" "$__messages_in"; then
+   printf 'apt-get update || apt-get update\n'
+fi
+

cdist/conf/type/__apt_source/man.rst

@@ -8,7 +8,8 @@ cdist-type__apt_source - Manage apt sources
 
 DESCRIPTION
 -----------
-This cdist type allows you to manage apt sources.
+This cdist type allows you to manage apt sources. It invokes index update
+internally when needed so call of index updating type is not needed.
 
 
 REQUIRED PARAMETERS
@@ -63,7 +64,7 @@ Steven Armstrong <steven-cdist--@--armstrong.cc>
 
 COPYING
 -------
-Copyright \(C) 2011-2014 Steven Armstrong. You can redistribute it
+Copyright \(C) 2011-2018 Steven Armstrong. You can redistribute it
 and/or modify it under the terms of the GNU General Public License as
 published by the Free Software Foundation, either version 3 of the
 License, or (at your option) any later version.

cdist/conf/type/__apt_source/manifest

@@ -1,6 +1,6 @@
 #!/bin/sh -e
 #
-# 2011-2013 Steven Armstrong (steven-cdist at armstrong.cc)
+# 2011-2018 Steven Armstrong (steven-cdist at armstrong.cc)
 #
 # This file is part of cdist.
 #
@@ -50,5 +50,3 @@ __file "/etc/apt/sources.list.d/${name}.list" \
    --source "$__object/files/source.list" \
    --owner root --group root --mode 0644 \
    --state "$state"
-
-require="$__object_name" __apt_update_index

cdist/conf/type/__apt_unattended_upgrades/man.rst

@@ -0,0 +1,68 @@
+cdist-type__apt_unattended_upgrades(7)
+======================================
+
+NAME
+----
+cdist-type__apt_unattended_upgrades - automatic installation of updates
+
+
+DESCRIPTION
+-----------
+
+Install and configure unattended-upgrades package.
+
+For more information see https://wiki.debian.org/UnattendedUpgrades.
+
+
+OPTIONAL MULTIPLE PARAMETERS
+----------------------------
+option
+   Set options for unattended-upgrades. See examples.
+
+   Supported options with default values (as of 2020-01-17) are:
+
+   - AutoFixInterruptedDpkg, default is "true"
+   - MinimalSteps, default is "true"
+   - InstallOnShutdown, default is "false"
+   - Mail, default is "" (empty)
+   - MailOnlyOnError, default is "false"
+   - Remove-Unused-Kernel-Packages, default is "true"
+   - Remove-New-Unused-Dependencies, default is "true"
+   - Remove-Unused-Dependencies, default is "false"
+   - Automatic-Reboot, default is "false"
+   - Automatic-Reboot-WithUsers, default is "true"
+   - Automatic-Reboot-Time, default is "02:00"
+   - SyslogEnable, default is "false"
+   - SyslogFacility, default is "daemon"
+   - OnlyOnACPower, default is "true"
+   - Skip-Updates-On-Metered-Connections, default is "true"
+   - Verbose, default is "false"
+   - Debug, default is "false"
+
+blacklist
+   Python regular expressions, matching packages to exclude from upgrading.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    __apt_unattended_upgrades \
+        --option Mail=root \
+        --option MailOnlyOnError=true \
+        --blacklist multipath-tools \
+        --blacklist open-iscsi
+
+
+AUTHORS
+-------
+Ander Punnar <ander-at-kvlt-dot-ee>
+
+
+COPYING
+-------
+Copyright \(C) 2020 Ander Punnar. You can redistribute it and/or modify it
+under the terms of the GNU General Public License as published by the Free
+Software Foundation, either version 3 of the License, or (at your option) any
+later version.

cdist/conf/type/__apt_unattended_upgrades/manifest

@@ -0,0 +1,80 @@
+#!/bin/sh -e
+#
+# 2020 Ander Punnar (ander-at-kvlt-dot-ee)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+__package unattended-upgrades
+
+export require='__package/unattended-upgrades'
+
+# in normal circumstances 20auto-upgrades is managed
+# by debconf and it can only contain these lines
+
+__file /etc/apt/apt.conf.d/20auto-upgrades \
+    --owner root \
+    --group root \
+    --mode 644 \
+    --source - << EOF
+APT::Periodic::Update-Package-Lists "1";
+APT::Periodic::Unattended-Upgrade "1";
+EOF
+
+# lets not write into upstream 50unattended-upgrades file,
+# but use our own config file to avoid clashes
+
+conf_file='/etc/apt/apt.conf.d/51unattended-upgrades-cdist'
+
+conf='# this file is managed by cdist'
+
+if [ -f "$__object/parameter/option" ]
+then
+    o=''
+
+    while read -r l
+    do
+        o="$( printf '%s\nUnattended-Upgrade::%s "%s";\n' "$o" "${l%%=*}" "${l#*=}" )"
+    done \
+        < "$__object/parameter/option"
+
+    conf="$( printf '%s\n%s\n' "$conf" "$o" )"
+fi
+
+if [ -f "$__object/parameter/blacklist" ]
+then
+    b='Unattended-Upgrade::Package-Blacklist {'
+
+    while read -r l
+    do
+        b="$( printf '%s\n"%s";\n' "$b" "$l" )"
+    done \
+        < "$__object/parameter/blacklist"
+
+    conf="$( printf '%s\n%s\n}\n' "$conf" "$b" )"
+fi
+
+if [ "$( echo "$conf" | wc -l )" -gt 1 ]
+then
+    echo "$conf" \
+        | __file "$conf_file" \
+            --owner root \
+            --group root \
+            --mode 644 \
+            --source -
+else
+    __file "$conf_file" --state absent
+fi

cdist/conf/type/__apt_unattended_upgrades/parameter/optional_multiple

@@ -0,0 +1,2 @@
+option
+blacklist

cdist/conf/type/__block/gencode-remote

@@ -18,6 +18,11 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
+# quote function from http://www.etalabs.net/sh_tricks.html
+quote() {
+   printf '%s\n' "$1" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/"
+}
+
 file="$(cat "$__object/parameter/file" 2>/dev/null || echo "/$__object_id")"
 state_should=$(cat "$__object/parameter/state")
 prefix=$(cat "$__object/parameter/prefix" 2>/dev/null || echo "#cdist:__block/$__object_id")
@@ -41,28 +46,29 @@ fi
 
 remove_block() {
    cat << DONE
-tmpfile=\$(mktemp ${file}.cdist.XXXXXXXXXX)
+tmpfile=\$(mktemp ${quoted_file}.cdist.XXXXXXXXXX)
 # preserve ownership and permissions of existing file
-if [ -f "$file" ]; then
-   cp -p "$file" "\$tmpfile"
+if [ -f $quoted_file ]; then
+   cp -p $quoted_file "\$tmpfile"
 fi
-awk -v prefix="^$prefix\$" -v suffix="^$suffix\$" '
+awk -v prefix=$(quote "$prefix") -v suffix=$(quote "$suffix") '
 {
-   if (match(\$0,prefix)) {
+   if (\$0 == prefix) {
       triggered=1
    }
    if (triggered) {
-      if (match(\$0,suffix)) {
+      if (\$0 == suffix) {
          triggered=0
       }
    } else {
       print
    }
-}' "$file" > "\$tmpfile"
-mv -f "\$tmpfile" "$file"
+}' $quoted_file > "\$tmpfile"
+mv -f "\$tmpfile" $quoted_file
 DONE
 }
 
+quoted_file="$(quote "$file")"
 case "$state_should" in
    present)
       if [ "$state_is" = "changed" ]; then
@@ -72,7 +78,7 @@ case "$state_should" in
          echo add >> "$__messages_out"
       fi
       cat << DONE
-cat >> "$file" << ${__type##*/}_DONE
+cat >> $quoted_file << '${__type##*/}_DONE'
 $(cat "$block")
 ${__type##*/}_DONE
 DONE

cdist/conf/type/__block/manifest

@@ -18,8 +18,6 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-
-file="$(cat "$__object/parameter/file" 2>/dev/null || echo "/$__object_id")"
 prefix=$(cat "$__object/parameter/prefix" 2>/dev/null || echo "#cdist:__block/$__object_id")
 suffix=$(cat "$__object/parameter/suffix" 2>/dev/null || echo "#/cdist:__block/$__object_id")
 text=$(cat "$__object/parameter/text")

cdist/conf/type/__ccollect_source/gencode-remote

@@ -42,21 +42,20 @@ get_current_value() {
 }
 
 set_group() {
-    echo chgrp \"$1\" \"$destination\"
-    echo chgrp $1 >> "$__messages_out"
+    echo "chgrp '$1' '$destination'"
+    echo "chgrp '$1'" >> "$__messages_out"
 }
 
 set_owner() {
-    echo chown \"$1\" \"$destination\"
-    echo chown $1 >> "$__messages_out"
+    echo "chown '$1' '$destination'"
+    echo "chown '$1'" >> "$__messages_out"
 }
 
 set_mode() {
-   echo chmod \"$1\" \"$destination\"
-   echo chmod $1 >> "$__messages_out"
+   echo "chmod '$1' '$destination'"
+   echo "chmod '$1'" >> "$__messages_out"
 }
 
-set_attributes=
 case "$state_should" in
     present|exists)
     # Note: Mode - needs to happen last as a chown/chgrp can alter mode by
@@ -67,11 +66,11 @@ case "$state_should" in
 
             # change 0xxx format to xxx format => same as stat returns
             if [ "$attribute" = mode ]; then
-                value_should="$(echo $value_should | sed 's/^0\(...\)/\1/')"
+                value_should="$(echo "$value_should" | sed 's/^0\(...\)/\1/')"
             fi
             
             value_is="$(get_current_value "$attribute" "$value_should")"
-            if [ -f "$__object/files/set-attributes" -o "$value_should" != "$value_is" ]; then
+            if [ -f "$__object/files/set-attributes" ] || [ "$value_should" != "$value_is" ]; then
                 "set_$attribute" "$value_should"
             fi
         fi
@@ -81,7 +80,7 @@ case "$state_should" in
 
     absent)
         if [ "$type" = "file" ]; then
-            echo rm -f \"$destination\"
+            echo "rm -f '$destination'"
             echo remove >> "$__messages_out"
         fi
     ;;

cdist/conf/type/__ccollect_source/man.rst

@@ -38,6 +38,8 @@ BOOLEAN PARAMETERS
 verbose
     Whether to report backup verbosely
 
+create-destination
+    Create the directory specified in the destination parameter on the remote host
 
 EXAMPLES
 --------
@@ -50,6 +52,13 @@ EXAMPLES
         --exclude '/proc/*' --exclude '/sys/*' \
         --verbose
 
+    __ccollect_source doc.ungleich.ch \
+        --source doc.ungleich.ch:/ \
+        --destination /backup/doc.ungleich.ch \
+        --exclude '/proc/*' --exclude '/sys/*' \
+        --verbose \ 
+        --create-destination
+
 
 SEE ALSO
 --------

cdist/conf/type/__ccollect_source/manifest

@@ -22,7 +22,7 @@ name="$__object_id"
 state="$(cat "$__object/parameter/state")"
 source="$(cat "$__object/parameter/source")"
 destination="$(cat "$__object/parameter/destination")"
-ccollectconf="$(cat "$__object/parameter/ccollectconf" | sed 's,/$,,')"
+ccollectconf="$(sed 's,/$,,' "$__object/parameter/ccollectconf")"
 
 sourcedir="$ccollectconf/sources"
 basedir="$sourcedir/$name"
@@ -53,3 +53,7 @@ if [ -f "$__object/parameter/exclude" ]; then
     __file "$exclude_file" --source - --state "$state" \
         < "$__object/parameter/exclude"
 fi
+
+if [ -f "$__object/parameter/create-destination" ]; then
+    __directory "${destination}" --parents --state "${state}"
+fi

cdist/conf/type/__ccollect_source/parameter/boolean

@@ -1 +1,2 @@
 verbose
+create-destination

cdist/conf/type/__cdist/man.rst

@@ -30,7 +30,7 @@ username
 
 source
     Select the source from which to clone cdist from.
-    Defaults to "git://github.com/ungleich/cdist.git".
+    Defaults to "git@code.ungleich.ch:ungleich-public/cdist.git".
 
 
 branch
@@ -47,7 +47,7 @@ EXAMPLES
     __cdist /home/cdist/cdist
 
     # Use alternative source
-    __cdist --source "git://github.com/ungleich/cdist" /home/cdist/cdist
+    __cdist --source "git@code.ungleich.ch:ungleich-public/cdist.git" /home/cdist/cdist
 
 
 AUTHORS

cdist/conf/type/__cdist/manifest

@@ -37,6 +37,7 @@ source="$(cat "$__object/parameter/source")"
 # out of it
 home=/home/$username
 
+# shellcheck disable=SC2086
 __user "$username" --home "$home" $shell
 
 require="__user/$username" __directory "$home" \

cdist/conf/type/__cdist/parameter/default/source

@@ -1 +1 @@
-git://github.com/ungleich/cdist.git
+git@code.ungleich.ch:ungleich-public/cdist.git

cdist/conf/type/__check_messages/gencode-remote

@@ -0,0 +1,26 @@
+#!/bin/sh -e
+#
+# 2019 Ander Punnar (ander-at-kvlt-dot-ee)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+if grep -Eq \
+    "$( cat "$__object/parameter/pattern" )" \
+    "$__messages_in"
+then
+    tee "$__messages_out" < "$__object/parameter/execute"
+fi

cdist/conf/type/__check_messages/man.rst

@@ -0,0 +1,52 @@
+cdist-type__check_messages(7)
+=============================
+
+NAME
+----
+cdist-type__check_messages - Check messages for pattern and execute command on match.
+
+
+DESCRIPTION
+-----------
+Check messages for pattern and execute command on match.
+
+This type is useful if you chain together multiple related types using
+dependencies and want to restart service if at least one type changes
+something.
+
+For more information about messages see `cdist messaging <cdist-messaging.html>`_.
+
+For more information about dependencies and execution order see
+`cdist manifest <cdist-manifest.html#dependencies>`_ documentation.
+
+
+REQUIRED PARAMETERS
+-------------------
+pattern
+   Extended regular expression pattern for search (passed to ``grep -E``).
+
+execute
+   Command to execute on pattern match.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    __check_messages munin \
+        --pattern '^__(file|link|line)/etc/munin/' \
+        --execute 'service munin-node restart'
+
+
+AUTHORS
+-------
+Ander Punnar <ander-at-kvlt-dot-ee>
+
+
+COPYING
+-------
+Copyright \(C) 2019 Ander Punnar. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.

cdist/conf/type/__check_messages/parameter/required

@@ -0,0 +1,2 @@
+pattern
+execute

cdist/conf/type/__clean_path/explorer/list

@@ -0,0 +1,40 @@
+#!/bin/sh -e
+#
+# 2019 Ander Punnar (ander-at-kvlt-dot-ee)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+if [ -f "$__object/parameter/path" ]
+then
+    path="$( cat "$__object/parameter/path" )"
+else
+    path="/$__object_id"
+fi
+
+[ ! -d "$path" ] && exit 0
+
+pattern="$( cat "$__object/parameter/pattern" )"
+
+if [ -f "$__object/parameter/exclude" ]
+then
+    exclude="$( cat "$__object/parameter/exclude" )"
+
+    find "$path" -mindepth 1 -maxdepth 1 -regex "$pattern" \
+        -and -not -regex "$exclude"
+else
+    find "$path" -mindepth 1 -maxdepth 1 -regex "$pattern"
+fi

cdist/conf/type/__clean_path/gencode-remote

@@ -0,0 +1,53 @@
+#!/bin/sh -e
+#
+# 2019 Ander Punnar (ander-at-kvlt-dot-ee)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+[ ! -s "$__object/explorer/list" ] && exit 0
+
+if [ -f "$__object/parameter/path" ]
+then
+    path="$( cat "$__object/parameter/path" )"
+else
+    path="/$__object_id"
+fi
+
+pattern="$( cat "$__object/parameter/pattern" )"
+
+if [ -f "$__object/parameter/exclude" ]
+then
+    exclude="$( cat "$__object/parameter/exclude" )"
+
+    echo "find '$path' -mindepth 1 -maxdepth 1 -regex '$pattern'" \
+        "-and -not -regex '$exclude'" \
+        '-exec rm -rf {} \;'
+else
+    echo "find '$path' -mindepth 1 -maxdepth 1 -regex '$pattern'" \
+        '-exec rm -rf {} \;'
+fi
+
+while read -r f
+do
+    echo "removed '$f'" >> "$__messages_out"
+done \
+< "$__object/explorer/list"
+
+if [ -f "$__object/parameter/onchange" ]
+then
+    cat "$__object/parameter/onchange"
+fi

cdist/conf/type/__clean_path/man.rst

@@ -0,0 +1,68 @@
+cdist-type__clean_path(7)
+=========================
+
+NAME
+----
+cdist-type__clean_path - Remove files and directories which match the pattern.
+
+
+DESCRIPTION
+-----------
+Remove files and directories which match the pattern.
+
+Provided path must be a directory.
+
+Patterns are passed to ``find``'s ``-regex`` - see ``find(1)`` for more details.
+
+Look up of files and directories is non-recursive (``-maxdepth 1``).
+
+Parent directory is excluded (``-mindepth 1``).
+
+This type is not POSIX compatible (sorry, Solaris users).
+
+
+REQUIRED PARAMETERS
+-------------------
+pattern
+   Pattern of files which are removed from path.
+
+
+OPTIONAL PARAMETERS
+-------------------
+path
+   Path which will be cleaned. Defaults to ``$__object_id``.
+
+exclude
+   Pattern of files which are excluded from removal.
+
+onchange
+   The code to run if files or directories were removed.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    __clean_path /etc/apache2/conf-enabled \
+        --pattern '.+' \
+        --exclude '.+\(charset\.conf\|security\.conf\)' \
+        --onchange 'service apache2 restart'
+
+    __clean_path apache2-conf-enabled \
+        --path /etc/apache2/conf-enabled \
+        --pattern '.+' \
+        --exclude '.+\(charset\.conf\|security\.conf\)' \
+        --onchange 'service apache2 restart'
+
+AUTHORS
+-------
+Ander Punnar <ander-at-kvlt-dot-ee>
+
+
+COPYING
+-------
+Copyright \(C) 2019 Ander Punnar. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.

cdist/conf/type/__clean_path/parameter/optional

@@ -0,0 +1,3 @@
+exclude
+onchange
+path

cdist/conf/type/__clean_path/parameter/required

@@ -0,0 +1 @@
+pattern

cdist/conf/type/__config_file/gencode-remote

@@ -19,16 +19,9 @@
 #
 
 destination="$__object_id"
-state="$(cat "$__object/parameter/state")"
-
-if [ "$state" = "absent" ]; then
-   # nothing to do
-   exit 0
-fi
 
 if [ -f "$__object/parameter/onchange" ]; then
    if grep -q "^__file/${destination}" "$__messages_in"; then
       cat "$__object/parameter/onchange"
    fi
 fi
-

cdist/conf/type/__config_file/manifest

@@ -19,7 +19,8 @@
 #
 
 set -- "/${__object_id}"
-for param in $(ls "$__object/parameter/"); do
+cd "$__object/parameter/"
+for param in *; do
    case "$param" in
       source)
          source="$(cat "$__object/parameter/source")"

cdist/conf/type/__consul/files/versions/1.0.6/cksum

@@ -0,0 +1 @@
+4120550353 48801129 consul

cdist/conf/type/__consul/files/versions/1.0.6/source

@@ -0,0 +1 @@
+https://releases.hashicorp.com/consul/1.0.6/consul_1.0.6_linux_amd64.zip

cdist/conf/type/__consul/files/versions/1.2.3/cksum

@@ -0,0 +1 @@
+191982 110369685

cdist/conf/type/__consul/files/versions/1.2.3/source

@@ -0,0 +1 @@
+https://releases.hashicorp.com/consul/1.2.3/consul_1.2.3_linux_amd64.zip

cdist/conf/type/__consul/files/versions/1.3.0/cksum

@@ -0,0 +1 @@
+1714523667 98363467 consul

cdist/conf/type/__consul/files/versions/1.3.0/source

@@ -0,0 +1 @@
+https://releases.hashicorp.com/consul/1.3.0/consul_1.3.0_linux_amd64.zip

cdist/conf/type/__consul/files/versions/1.5.0/cksum

@@ -0,0 +1 @@
+886614099 103959898 consul

cdist/conf/type/__consul/files/versions/1.5.0/source

@@ -0,0 +1 @@
+https://releases.hashicorp.com/consul/1.5.0/consul_1.5.0_linux_amd64.zip

cdist/conf/type/__consul/gencode-remote

@@ -0,0 +1,63 @@
+#!/bin/sh -e
+#
+# 2018 Darko Poljak (darko.poljak at gmail.com)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+#set -x
+
+if [ ! -f "$__object/parameter/direct" ]; then
+    # Nothing here, staged file is used.
+    exit 0
+fi
+
+state=$(cat "$__object/parameter/state")
+destination="/usr/local/bin/consul"
+
+if [ "$state" = "absent" ]; then
+    printf 'rm -f "%s"' "$destination"
+    exit 0
+fi
+
+versions_dir="$__type/files/versions"
+version="$(cat "$__object/parameter/version")"
+version_dir="$versions_dir/$version"
+
+source=$(cat "$version_dir/source")
+source_file_name="${source##*/}"
+cksum_should=$(cut -d' ' -f1,2 "$version_dir/cksum")
+
+cat << eof
+    tmpdir=\$(mktemp -d -p /tmp "${__type##*/}.XXXXXXXXXX")
+    curl -s -L "$source" > "\$tmpdir/$source_file_name"
+    unzip -p "\$tmpdir/$source_file_name" > "${destination}.tmp"
+    rm -rf "\$tmpdir"
+
+    cksum_is=\$(cksum "${destination}.tmp" | cut -d' ' -f1,2)
+    if [ "\$cksum_is" = "$cksum_should" ]; then
+        rm -f "${destination}"
+        mv "${destination}.tmp" "${destination}"
+        chown root:root "$destination"
+        chmod 755 "$destination"
+    else
+        rm -f "${destination}.tmp"
+        echo "Failed to verify checksum for $__object_name" >&2
+        exit 1
+    fi
+eof
+
+echo "/usr/local/bin/consul created" >> "$__messages_out"