Apply machine_type explorer fix from pedro

e139397529

bin/cdist-build-helper

@@ -1,6 +1,6 @@
 #!/bin/sh
 #
-# 2011-2013 Nico Schottelius (nico-cdist at schottelius.org)
+# 2011-2022 Nico Schottelius (nico-cdist at schottelius.org)
 # 2016-2019 Darko Poljak (darko.poljak at gmail.com)
 #
 # This file is part of cdist.

cdist/argparse.py

@@ -472,9 +472,6 @@ def get_parsers():
     parser['info'].set_defaults(func=cdist.info.Info.commandline)
 
     # Scan = config + further
-    parser['scan'] = parser['sub'].add_parser('scan', add_help=False,
-                                              parents=[parser['config']])
-
     parser['scan'] = parser['sub'].add_parser(
             'scan', parents=[parser['loglevel'],
                              parser['beta'],

cdist/conf/explorer/lsb_codename

@@ -21,6 +21,9 @@
 
 set +e
 case "$("$__explorer/os")" in
+   checkpoint)
+       awk '{printf("%s\n", $(NF-1))}' /etc/cp-release
+   ;;
    openwrt)
       # shellcheck disable=SC1091
       (. /etc/openwrt_release && echo "$DISTRIB_CODENAME")

cdist/conf/explorer/lsb_description

@@ -21,6 +21,9 @@
 
 set +e
 case "$("$__explorer/os")" in
+   checkpoint)
+       cat /etc/cp-release
+   ;;
    openwrt)
       # shellcheck disable=SC1091
       (. /etc/openwrt_release && echo "$DISTRIB_DESCRIPTION")

cdist/conf/explorer/lsb_id

@@ -21,6 +21,9 @@
 
 set +e
 case "$("$__explorer/os")" in
+   checkpoint)
+       echo "CheckPoint"
+   ;;
    openwrt)
       # shellcheck disable=SC1091
       (. /etc/openwrt_release && echo "$DISTRIB_ID")

cdist/conf/explorer/lsb_release

@@ -21,6 +21,9 @@
 
 set +e
 case "$("$__explorer/os")" in
+   checkpoint)
+       sed /etc/cp-release -e 's/.* R\([1-9][0-9]*\)\.[0-9]*$/\1/'
+   ;;
    openwrt)
       # shellcheck disable=SC1091
       (. /etc/openwrt_release && echo "$DISTRIB_RELEASE")

cdist/conf/explorer/machine_type

@@ -515,7 +515,7 @@ check_vm_arch_specific() {
 				&& return 0
 			fi
 			if has_cpuinfo \
-				&& grep -q -i -e '^flags.*:.*\(hypervisor\|vmm\)' /proc/cpuinfo
+				&& grep -q -i -e '^flags.*:.*\(hypervisor\|vmm\)\( \|$\)' /proc/cpuinfo
 			then
 				return 0
 			fi

cdist/conf/explorer/os

@@ -116,6 +116,13 @@ if [ -f /etc/slackware-version ]; then
    exit 0
 fi
 
+# Appliances
+
+if grep -q '^Check Point Gaia' /etc/cp-release 2>/dev/null; then
+    echo checkpoint
+    exit 0
+fi
+
 uname_s="$(uname -s)"
 
 # Assume there is no tr on the client -> do lower case ourselves

cdist/conf/explorer/os_release

@@ -34,5 +34,9 @@ elif test -f /var/run/os-release
 then
 	# FreeBSD (created by os-release service)
 	cat /var/run/os-release
+elif test -f /etc/cp-release
+then
+        # Checkpoint firewall or management (actually linux based)
+        cat /etc/cp-release
 fi
 

cdist/conf/explorer/os_version

@@ -41,6 +41,9 @@ in
       # empty, but well...
       cat /etc/arch-release
    ;;
+   checkpoint)
+       awk '{version=$NF; printf("%s\n", substr(version, 2))}' /etc/cp-release
+    ;;
    debian)
       debian_version=$(cat /etc/debian_version)
       case $debian_version
@@ -79,6 +82,7 @@ in
             # ceres versions don't have a number, so we decode by codename:
             case ${devuan_version}
             in
+               (daedalus/ceres) echo 4.99 ;;
                (chimaera/ceres) echo 3.99 ;;
                (beowulf/ceres) echo 2.99 ;;
                (ascii/ceres) echo 1.99 ;;

cdist/conf/type/__apt_mark/explorer/state

@@ -24,4 +24,4 @@ else
     name="$__object_id"
 fi
 
-apt-mark showhold | grep -Fq "$name" && echo hold || echo unhold
+apt-mark showhold | grep -q "^${name}$" && echo hold || echo unhold

cdist/conf/type/__apt_pin/manifest

@@ -57,6 +57,11 @@ __file "/etc/apt/preferences.d/$name" \
     --owner root --group root --mode 0644 \
     --state "$state" \
     --source - << EOF
+# Created by cdist ${__type##*/}
+# Do not change. Changes will be overwritten.
+#
+
+# $name
 Package: $package
 Pin: $pin
 Pin-Priority: $priority

cdist/conf/type/__apt_pin/parameter/default/priority

@@ -0,0 +1 @@
+500

cdist/conf/type/__apt_pin/parameter/optional

@@ -1,2 +1,3 @@
 state
 package
+priority

cdist/conf/type/__apt_pin/parameter/required

@@ -1,2 +1 @@
 distribution
-priority

cdist/conf/type/__apt_ppa/files/remove-apt-repository

@@ -1,55 +0,0 @@
-#!/usr/bin/env python
-#
-# Remove the given apt repository.
-#
-# Exit with:
-#   0: if it worked
-#   1: if not
-#   2: on other error
-
-import os
-import sys
-from aptsources import distro, sourceslist
-from softwareproperties import ppa
-from softwareproperties.SoftwareProperties import SoftwareProperties
-
-
-def remove_if_empty(file_name):
-    with open(file_name, 'r') as f:
-        if f.read().strip():
-            return
-        os.unlink(file_name)
-
-def remove_repository(repository):
-    #print 'repository:', repository
-    codename = distro.get_distro().codename
-    #print 'codename:', codename
-    (line, file) = ppa.expand_ppa_line(repository.strip(), codename)
-    #print 'line:', line
-    #print 'file:', file
-    deb_source_entry = sourceslist.SourceEntry(line, file)
-    src_source_entry = sourceslist.SourceEntry('deb-src{}'.format(line[3:]), file)
-
-    try:
-        sp = SoftwareProperties()
-        sp.remove_source(deb_source_entry)
-        try:
-            # If there's a deb-src entry, remove that too
-            sp.remove_source(src_source_entry)
-        except:
-            pass
-        remove_if_empty(file)
-        return True
-    except ValueError:
-        print >> sys.stderr, "Error: '%s' doesn't exists in a sourcelist file" % line
-        return False
-
-if __name__ == '__main__':
-    if (len(sys.argv) != 2):
-        print >> sys.stderr, 'Error: need a repository as argument'
-        sys.exit(2)
-    repository = sys.argv[1]
-    if remove_repository(repository):
-        sys.exit(0)
-    else:
-        sys.exit(1)

cdist/conf/type/__apt_ppa/gencode-remote

@@ -29,9 +29,9 @@ fi
 
 case "$state_should" in
    present)
-      echo "add-apt-repository '$name'"
+      echo "add-apt-repository -y '$name'"
    ;;
    absent)
-      echo "remove-apt-repository '$name'"
+      echo "add-apt-repository -r -y '$name'"
    ;;
 esac

cdist/conf/type/__apt_ppa/manifest

@@ -20,9 +20,4 @@
 
 __package software-properties-common
 
-require="__package/software-properties-common" \
-   __file /usr/local/bin/remove-apt-repository \
-   --source "$__type/files/remove-apt-repository" \
-   --mode 0755
-
 require="$__object_name" __apt_update_index

cdist/conf/type/__apt_source/files/source.list.template

@@ -2,13 +2,14 @@
 set -u
 
 entry="$uri $distribution $component"
+
 cat << DONE
 # Created by cdist ${__type##*/}
 # Do not change. Changes will be overwritten.
 #
 
 # $name
-deb ${forcedarch} $entry
+deb ${options} $entry
 DONE
 if [ -f "$__object/parameter/include-src" ]; then
    echo "deb-src $entry"

cdist/conf/type/__apt_source/man.rst

@@ -23,6 +23,9 @@ OPTIONAL PARAMETERS
 arch
    set this if you need to force and specific arch (ubuntu specific)
 
+signed-by
+   provide a GPG key fingerprint or keyring path for signature checks
+
 state
    'present' or 'absent', defaults to 'present'
 
@@ -56,6 +59,11 @@ EXAMPLES
        --uri http://archive.canonical.com/ \
        --component partner --state present
 
+    __apt_source goaccess \
+       --uri http://deb.goaccess.io/ \
+       --component main \
+       --signed-by C03B48887D5E56B046715D3297BD1A0133449C3D
+
 
 AUTHORS
 -------

cdist/conf/type/__apt_source/manifest

@@ -21,6 +21,7 @@
 name="$__object_id"
 state="$(cat "$__object/parameter/state")"
 uri="$(cat "$__object/parameter/uri")"
+options=""
 
 if [ -f "$__object/parameter/distribution" ]; then
    distribution="$(cat "$__object/parameter/distribution")"
@@ -31,9 +32,15 @@ fi
 component="$(cat "$__object/parameter/component")"
 
 if [ -f "$__object/parameter/arch" ]; then
-   forcedarch="[arch=$(cat "$__object/parameter/arch")]"
-else
-   forcedarch=""
+   options="arch=$(cat "$__object/parameter/arch")"
+fi
+
+if [ -f "$__object/parameter/signed-by" ]; then
+   options="$options signed-by=$(cat "$__object/parameter/signed-by")"
+fi
+
+if [ "$options" ]; then
+    options="[$options]"
 fi
 
 # export variables for use in template
@@ -41,7 +48,7 @@ export name
 export uri
 export distribution
 export component
-export forcedarch
+export options
 
 # generate file from template
 mkdir "$__object/files"

cdist/conf/type/__apt_source/parameter/optional

@@ -1,4 +1,5 @@
 state
 distribution
 component
-arch
+arch
+signed-by

cdist/conf/type/__dot_file/man.rst

@@ -37,6 +37,12 @@ state
 source
     forwarded to :strong:`__file` type
 
+file
+    forwarded to :strong:`__file` type
+    This can be used if multiple users need to have a dotfile updated,
+    which will result in duplicate object id errors. When using the
+    file parameter the object id can be some unique value.
+
 MESSAGES
 --------
 
@@ -61,6 +67,15 @@ EXAMPLES
     # Install default xmonad config for user 'eve'. Parent directory is created automatically.
     __dot_file .xmonad/xmonad.hs --user eve --state exists --source "$__files/xmonad.hs"
 
+    # install .vimrc for root and some users
+    for user in root userx usery userz; do
+        __dot_file "${user}_dot_vimrc" \
+            --user $user \
+            --file .vimrc \
+            --state exists \
+            --source "$__files/$user/.vimrc"
+    done
+
 SEE ALSO
 --------
 

cdist/conf/type/__dot_file/manifest

@@ -20,13 +20,19 @@ user="$(cat "${__object}/parameter/user")"
 home="$(cat "${__object}/explorer/home")"
 primary_group="$(cat "${__object}/explorer/primary_group")"
 dirmode="$(cat "${__object}/parameter/dirmode")"
+if [ -f "${__object}/parameter/file" ]; then
+    file="$(cat "${__object}/parameter/file")"
+else
+    file="${__object_id}"
+fi
+
 
 # Create parent directory. Type __directory has flag 'parents', but it
 # will leave us with root-owned directory in user home, which is not
 # acceptable. So we create parent directories one-by-one. XXX: maybe
 # it should be fixed in '__directory'?
 set --
-subpath=${__object_id}
+subpath=${file}
 while subpath="$(dirname "${subpath}")" ; do
 	[ "${subpath}" = . ] && break
 	set -- "${subpath}" "$@"
@@ -64,4 +70,4 @@ if [ "${source}" = "-" ] ; then
 fi
 unset source
 
-__file "${home}/${__object_id}" --owner "$user" --group "$primary_group" "$@"
+__file "${home}/${file}" --owner "$user" --group "$primary_group" "$@"

cdist/conf/type/__file/gencode-local

@@ -1,7 +1,7 @@
 #!/bin/sh -e
 #
 # 2011-2012 Nico Schottelius (nico-cdist at schottelius.org)
-# 2013 Steven Armstrong (steven-cdist armstrong.cc)
+# 2013-2022 Steven Armstrong (steven-cdist armstrong.cc)
 #
 # This file is part of cdist.
 #
@@ -72,6 +72,7 @@ if [ "$state_should" = "present" ] || [ "$state_should" = "exists" ]; then
          if [ "$type" != "file" ]; then
             # destination is not a regular file, upload source to replace it
             upload_file=1
+            echo upload >> "$__messages_out"
          else
             local_cksum="$(cksum < "$source")"
             remote_cksum="$(cat "$__object/explorer/cksum")"
@@ -88,27 +89,39 @@ if [ "$state_should" = "present" ] || [ "$state_should" = "exists" ]; then
       mkdir "$__object/files"
       touch "$__object/files/set-attributes"
 
-      # upload file to temp location
-      tempfile_template="${destination}.cdist.XXXXXXXXXX"
-      cat << DONE
-destination_upload="\$($__remote_exec $__target_host "mktemp $tempfile_template")"
-DONE
-      if [ "$upload_file" ]; then
-         echo upload >> "$__messages_out"
-         # IPv6 fix
-         if echo "${__target_host}" | grep -q -E '^[0-9a-fA-F:]+$'
-         then
-             my_target_host="[${__target_host}]"
-         else
-             my_target_host="${__target_host}"
-         fi
-         cat << DONE
-$__remote_copy "$source" "${my_target_host}:\$destination_upload"
-DONE
+      if [ "$create_file" ]; then
+         # When creating an empty file we create it locally and then
+         # upload it so that permissions can be set before moving the file
+         # into place.
+         source="$__object/files/empty"
+         touch "$source"
       fi
-# move uploaded file into place
-cat << DONE
-$__remote_exec $__target_host "rm -rf \"$destination\"; mv \"\$destination_upload\" \"$destination\""
+
+      # upload file to temp location
+      upload_destination="${destination}.cdist.${__cdist_object_marker}.$$"
+      # Yes, we are aware that this is a race condition.
+      # However:
+      # a) cdist usually writes to directories that are not user writable
+      #    (probably > 99.9%)
+      # b) if they are user owned, the user / attacker always wins
+      #    (probably < 0.1%)
+      # c) the only case which we could improve are tmp directories and we
+      #    don't think managing tmp directories with cdist is a typical case
+      #    ("the rest %)"
+
+      # Tell gencode-remote to where we uploaded the file so it can move
+      # it to its final destination.
+      echo "$upload_destination" > "$__object/files/upload-destination"
+
+      # IPv6 fix
+      if echo "${__target_host}" | grep -q -E '^[0-9a-fA-F:]+$'
+      then
+          my_target_host="[${__target_host}]"
+      else
+          my_target_host="${__target_host}"
+      fi
+      cat << DONE
+$__remote_copy "$source" "${my_target_host}:${upload_destination}"
 DONE
    fi
 fi

cdist/conf/type/__file/gencode-remote

@@ -1,7 +1,7 @@
 #!/bin/sh -e
 #
 # 2011-2013 Nico Schottelius (nico-cdist at schottelius.org)
-# 2013 Steven Armstrong (steven-cdist armstrong.cc)
+# 2013-2022 Steven Armstrong (steven-cdist armstrong.cc)
 #
 # This file is part of cdist.
 #
@@ -62,6 +62,13 @@ set_mode() {
 
 case "$state_should" in
     present|exists)
+        if [ -f "$__object/files/upload-destination" ]; then
+            final_destination="$destination"
+            # We change the 'global' $destination variable here so we can
+            # change attributes of the new/uploaded file before moving it
+            # to it's final destination.
+            destination="$(cat "$__object/files/upload-destination")"
+        fi
         # Note: Mode - needs to happen last as a chown/chgrp can alter mode by
         #  clearing S_ISUID and S_ISGID bits (see chown(2))
         for attribute in group owner mode; do
@@ -81,6 +88,11 @@ case "$state_should" in
                 fi
             fi
         done
+        if [ -f "$__object/files/upload-destination" ]; then
+            # move uploaded file into place
+            printf 'rm -rf "%s"\n' "$final_destination"
+            printf 'mv "%s" "%s"\n' "$destination" "$final_destination"
+        fi
         if [ -f "$__object/files/set-attributes" ]; then
             # set-attributes is created if file is created or uploaded in gencode-local
             fire_onchange=1

cdist/conf/type/__letsencrypt_cert/manifest

@@ -85,7 +85,7 @@ if [ -z "${certbot_fullpath}" ]; then
 			esac
 		;;
 		freebsd)
-			__package py37-certbot
+			__package py39-certbot
 			certbot_fullpath="/usr/local/bin/certbot"
 		;;
 		ubuntu)

cdist/conf/type/__ssh_authorized_key/gencode-remote

@@ -40,6 +40,7 @@ if [ -f "$file" ]; then
    grep -v -F -x '$line' '$file' >\$tmpfile
 fi
 cat "\$tmpfile" >"$file"
+rm -f "\$tmpfile"
 DONE
 }
 

docs/changelog

@@ -1,7 +1,15 @@
 Changelog
 ---------
 
-next:
+7.0.1:
+	* Core: Remove double definition of scan parser (Nico Schottelius)
+	* Type __apt_mark: Narrow down grep for hold packages (marcoduif)
+	* Type __apt_source: Set required options variable (Mark Verboom)
+	* Type __letsencrypt_cert: Update python version (Michelle)
+	* Explorer os_version: Add support for Daedalus (Michelle)
+	* Explorer machine_type: Correct incorrect VMM matching (Mark Verboom)
+
+7.0.0: 2022-07-31
 	* Explorer machine_type: Rewrite (Dennis Camera)
 	* New type: __sed (Ander Punnar)
 	* New type: __haproxy_dualstack (Evilham and ungleich)
@@ -12,6 +20,13 @@ next:
 	* Type __package_apt: Fix complaint about suite change (Matthias Stecher)
 	* Type __debconf_set_selections: Fix bug where --file was unsupported (Evilham)
 	* Types __letsencrypt_cert, __grafana_dashboard: Improve bullseye support (Evilham)
+	* Type __ssh_authorized_key: Also remove tmpfile if removing line (Mark Verboom)
+	* Type __apt_pin: Add default priority, add comment in generated files (Daniel Fancsali)
+	* Type __file: make file uploading and attribute changes more atomic	(Steven Armstrong)
+	* Type __dot_file: Add support for using --file parameter (Stephan Leemburg)
+	* Type __apt_ppa: Replace custom "remove-apt-repository" with add-apt-repository -r (Romain Dartigues)
+	* Type __apt_source: Add signed-by parameter (Daniel Fancsali)
+	* Explorer: add support for checkpoint (Stephan Leemburg)
 
 6.9.8: 2021-08-24
 	* Type __rsync: Rewrite (Ander Punnar)

docs/dev/release-process.org

@@ -0,0 +1,90 @@
+* Install requirements (Alpine)
+  - apk add py3-pycodestyle shellcheck py3-sphinx py3-sphinx_rtd_theme \
+    py3-build twine
+* Ensure your gpg setup works with the email used in the git commit!
+  - For me this is nico@nico-notebook.schottelius.org
+  - Signature / id is on nb2
+* Create ~/.pypirc
+[distutils]
+  index-servers =
+    pypi
+    cdist
+
+[pypi]
+  username = __token__
+  password = ...
+
+[cdist]
+  repository = https://upload.pypi.org/legacy/
+  username = __token__
+  password = ...
+
+* Add date in docs/changelog
+* Run ./bin/cdist-build-helper
+* TODO Move to "build"
+  - python3 -m build
+* DONE git tag: when?
+CLOSED: [2022-07-31 Sun 23:58]
+** Asked during release process: ok
+* DONE Pypi error with distutils: do not use distutils anymore
+CLOSED: [2022-07-31 Sun 23:58]
+python3 setup.py sdist upload
+...
+Creating tar archive
+removing 'cdist-7.0.0' (and everything under it)
+running upload
+Submitting dist/cdist-7.0.0.tar.gz to https://upload.pypi.org/legacy/
+Upload failed (400): Invalid value for blake2_256_digest. Error: Use a valid, hex-encoded, BLAKE2 message digest.
+error: Upload failed (400): Invalid value for blake2_256_digest. Error: Use a valid, hex-encoded, BLAKE2 message digest.
+(venv2) [22:50] nb2:cdist%
+
+* DONE Pypi error with twine: fixed in twine 4.0.1
+CLOSED: [2022-07-31 Sun 23:58]
+
+Seeing:
+
+(venv2) [22:47] nb2:cdist% twine upload dist/cdist-7.0.0*
+Uploading distributions to https://upload.pypi.org/legacy/
+Traceback (most recent call last):
+  File "/usr/bin/twine", line 8, in <module>
+    sys.exit(main())
+  File "/usr/lib/python3.10/site-packages/twine/__main__.py", line 28, in main
+    result = cli.dispatch(sys.argv[1:])
+  File "/usr/lib/python3.10/site-packages/twine/cli.py", line 68, in dispatch
+    return main(args.args)
+  File "/usr/lib/python3.10/site-packages/twine/commands/upload.py", line 197, in main
+    return upload(upload_settings, parsed_args.dists)
+  File "/usr/lib/python3.10/site-packages/twine/commands/upload.py", line 141, in upload
+    resp = repository.upload(package)
+  File "/usr/lib/python3.10/site-packages/twine/repository.py", line 189, in upload
+    resp = self._upload(package)
+  File "/usr/lib/python3.10/site-packages/twine/repository.py", line 144, in _upload
+    data = package.metadata_dictionary()
+  File "/usr/lib/python3.10/site-packages/twine/package.py", line 181, in metadata_dictionary
+    "dynamic": meta.dynamic,
+AttributeError: 'Wheel' object has no attribute 'dynamic'
+
+
+Fix:
+
+
+(venv2) [23:43] nb2:cdist% pipx run twine upload dist/*
+⚠️  twine is already on your PATH and installed at /home/nico/venv2/bin/twine. Downloading and running anyway.
+Uploading distributions to https://upload.pypi.org/legacy/
+Uploading cdist-7.0.0-py3-none-any.whl
+100% ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 868.6/868.6 kB • 00:04 • 221.3 kB/s
+Uploading cdist-7.0.0.tar.gz
+100% ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.5/1.5 MB • 00:08 • 169.3 kB/s
+
+View at:
+https://pypi.org/project/cdist/7.0.0/
+* TODO cdist web
+  - on staticweb-2022
+  - Should be moved to sftp/k8s
+
+
+  Manual steps:
+
+  ~/bin/permissions.public html/
+  rsync -a html/ staticweb.ungleich.ch:/home/services/www/nico/www.cdi.st/www/manual/7.0.0/
+  ssh staticweb.ungleich.ch "cd /home/services/www/nico/www.cdi.st/www/manual; ln -sf 7.0.0 latest"