Hexo 2025-12-09T17:34:35.000Z https://moki.org.pl/ Marcin Bagiński Hexo Update Debian 12 (Bookworm) to Debian 13 (Trixie) https://moki.org.pl/2025/12/09/update-debian-12-bookworm-to-13-trixie/ 2025-12-09T17:34:35.000Z 2025-12-09T17:34:35.000Z Prepare:

1
2
3
apt update
apt upgrade -qqy
apt dist-upgrade -qqy

Update APT:

1
sed -i 's/bookworm/trixie/g' /etc/apt/sources.list

Upgrade:

1
2
3
4
5
apt update && \
apt upgrade -qqy --without-new-pkgs && \
apt full-upgrade -qqy && \
apt --purge autoremove -qqy && \
apt clean

Reboot:

1
reboot
]]> <p>Prepare:</p> <div class="highlight-container" data-rel="Shell"><figure class="iseeu highlight shell"><table><tr><td class="gutter"><pre>< OpenStack Kolla - fix HAproxy https://moki.org.pl/2025/11/21/openstack-kolla-fix-haproxy/ 2025-11-21T16:50:34.000Z 2025-11-21T16:50:34.000Z 
1
2
3
ip addr add <kolla_external_vip_interface>/32 dev <iface>
ip addr add <kolla_internal_vip_interface>/32 dev <iface>
docker restart haproxy
]]> <div class="highlight-container" data-rel="Shell"><figure class="iseeu highlight shell"><table><tr><td class="gutter"><pre><span class="line Git - force use specific SSH key https://moki.org.pl/2025/11/15/git-force%20specific-ssh-key/ 2025-11-15T08:42:23.000Z 2025-11-15T08:42:23.000Z 
1
git config --local core.sshCommand 'ssh -i /home/user-name/.ssh/id_ed25519'
]]> <div class="highlight-container" data-rel="Shell"><figure class="iseeu highlight shell"><table><tr><td class="gutter"><pre><span class="line MySQL Group Replication - force group recovery https://moki.org.pl/2025/10/08/mysql-group-replication-force-group-recovery/ 2025-10-08T19:09:23.000Z 2025-10-08T19:09:23.000Z CLONE INSTANCE FROM is optional.

1
2
3
4
5
STOP GROUP_REPLICATION;
reset replica;
set global group_replication_clone_threshold=1;
CLONE INSTANCE FROM 'REPLICATION_USER'@'ALIVE_SERVER_IP':3306 IDENTIFIED BY 'REPLICATION_PASSWORD';
START GROUP_REPLICATION;
]]> <p><code>CLONE INSTANCE FROM</code> is optional.</p> <div class="highlight-container" data-rel="Sql"><figure class="iseeu highlight sql"><ta Ceph - remove host https://moki.org.pl/2025/09/25/ceph-remove-host/ 2025-09-25T05:29:01.000Z 2025-09-25T05:29:01.000Z List services:

1
ceph orch ps

Remove services:

1
ceph orch host drain NODE_NAME

Remove host:

1
ceph orch host rm NODE_NAME --force
]]> <p>List services:</p> <div class="highlight-container" data-rel="Shell"><figure class="iseeu highlight shell"><table><tr><td class="gutter"> Ceph - defragment OSD https://moki.org.pl/2025/09/15/ceph-defragment-osd/ 2025-09-15T05:19:01.000Z 2025-09-15T05:19:01.000Z Check for score:

1
ceph daemon osd.0 bluestore allocator score block

Stop OSD: 

1
ceph orch daemon stop osd.0

Deframgment:

1
cephadm shell --name osd.0 ceph-bluestore-tool bluestore trim

Start OSD:

1
ceph orch daemon start osd.0
]]> <p>Check for score:</p> <div class="highlight-container" data-rel="Shell"><figure class="iseeu highlight shell"><table><tr><td class="gutter OpenStack Kolla - remove RabbitMQ https://moki.org.pl/2025/09/03/openstack-kolla-remove-rabbitmq/ 2025-09-03T20:54:11.000Z 2025-09-03T20:54:11.000Z 
1
2
3
4
5
6
docker ps -a --filter "name=rabbitmq" -q | xargs -r docker rm -f ; \
docker volume ls --filter "name=rabbitmq" -q | xargs -r docker volume rm ; \
docker images --filter "reference=*rabbitmq*" -q | xargs -r docker rmi -f ; \
rm -rf /var/lib/kolla/volumes/rabbitmq ; \
rm -rf /etc/kolla/rabbitmq ; \
rm -rf /var/log/kolla/rabbitmq/
]]> <div class="highlight-container" data-rel="Shell"><figure class="iseeu highlight shell"><table><tr><td class="gutter"><pre><span class="line Ubuntu - grow root partition https://moki.org.pl/2025/08/20/ubuntu-grow-root-disk/ 2025-08-20T17:34:35.000Z 2025-08-20T17:34:35.000Z 
1
echo 1 | sudo tee /sys/class/block/sda/device/rescan ; parted -fs /dev/sda print && parted -sf /dev/sda resizepart 3 100% && partprobe && pvresize /dev/sda3 && lvextend -r -l +100%FREE /dev/ubuntu-vg/ubuntu-lv ; df -h
]]> <div class="highlight-container" data-rel="Shell"><figure class="iseeu highlight shell"><table><tr><td class="gutter"><pre><span class="line Ubuntu upgrade https://moki.org.pl/2025/08/14/ubuntu-upgrade/ 2025-08-14T05:19:01.000Z 2025-08-14T05:19:01.000Z 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
alias X="ssh root@1.2.3.4"
X <<'EOF'
tee /etc/apt/apt.conf.d/99force-conf > /dev/null <<'BLOB'
Dpkg::Options {
   "--force-confdef";
   "--force-confold";
}
BLOB
export DEBIAN_FRONTEND=noninteractive
apt update
apt update -y
apt full-upgrade -y
apt autoremove -y
exit
EOF
X -c "'reboot'"
sleep 120 && X -c "'export DEBIAN_FRONTEND=noninteractive && do-release-upgrade'"
]]> <div class="highlight-container" data-rel="Shell"><figure class="iseeu highlight shell"><table><tr><td class="gutter"><pre><span class="line OpenStack Kolla - fix MariaDB https://moki.org.pl/2025/08/06/openstack-kolla-fix-mariadb/ 2025-08-06T16:50:34.000Z 2025-08-06T16:50:34.000Z 
1
kolla-ansible mariadb-recovery -i /etc/kolla/inventory
]]> <div class="highlight-container" data-rel="Shell"><figure class="iseeu highlight shell"><table><tr><td class="gutter"><pre><span class="line OpenStack Kolla - fix Redis https://moki.org.pl/2025/08/06/openstack-kolla-fix-redis/ 2025-08-06T16:50:34.000Z 2025-08-06T16:50:34.000Z 
1
2
3
4
5
6
7
8
9
10
11
12
if [ "$(tail -n 100 /var/log/kolla/redis/redis.log | grep 'AOF file, then use ./redis-check-aof' | wc -l)" -gt 0 ]; then
    echo 'finxing'
    FILENAME="$(tail -n 100 /var/log/kolla/redis/redis.log | grep 'Bad file format reading the append only file' | head -n 1 | cut -d ':' -f 4 | cut -d ' ' -f 11)"
    IMAGE_ID="$(docker ps -a | grep 'kolla/redis:' | awk '{print $1}')"
    DOCKER_IMAGE_NAME="$(docker ps -a | grep "$IMAGE_ID" | awk '{print $2}')"
    docker stop redis
    docker run -it --rm -v /var/lib/docker/volumes/redis/_data/:/var/lib/redis $DOCKER_IMAGE_NAME bash -c "echo 'y' | redis-check-aof --fix '/var/lib/redis/appendonlydir/$FILENAME'"
    docker restart redis
    tail -f /var/log/kolla/redis/redis*.log
else
    echo 'problem not found'
fi
]]> <div class="highlight-container" data-rel="Shell"><figure class="iseeu highlight shell"><table><tr><td class="gutter"><pre><span class="line OpenStack Kolla - upgrade https://moki.org.pl/2025/08/06/openstack-kolla-upgrade/ 2025-08-06T16:50:34.000Z 2025-08-06T16:50:34.000Z Prepare passwords and inventory:

1
2
3
4
5
6
7
cp /usr/local/share/kolla-ansible/etc_examples/kolla/globals.yml /etc/kolla/globals.yml && \
cp /usr/local/share/kolla-ansible/ansible/inventory/multinode /etc/kolla/inventory/config/openstack && \
cp /etc/kolla/passwords.yml /tmp/passwords.yml.old && \
cp /usr/local/share/kolla-ansible/etc_examples/kolla/passwords.yml /tmp/passwords.yml.new && \
kolla-genpwd -p /tmp/passwords.yml.new && \
kolla-mergepwd --old /tmp/passwords.yml.old --new /tmp/passwords.yml.new --final /etc/kolla/passwords.yml && \
rm /tmp/passwords.yml.old /tmp/passwords.yml.new
  • diff /etc/kolla/globals.yml with your version from version control
  • diff /etc/kolla/inventory with your version from version control

Fix python:

1
rm -rf /usr/lib/python3.*/EXTERNALLY-MANAGED

Pull images:

1
kolla-ansible pull -i /etc/kolla/inventory

Checks:

1
kolla-ansible prechecks -i /etc/kolla/inventory

Upgrade:

1
kolla-ansible upgrade -i /etc/kolla/inventory

Post deploy:

1
kolla-ansible post-deploy -i /etc/kolla/inventory
]]> <p>Prepare passwords and inventory:</p> <div class="highlight-container" data-rel="Shell"><figure class="iseeu highlight shell"><table><tr>< Ceph - recovery speedup https://moki.org.pl/2025/08/06/ceph-recovery-speedup/ 2025-08-06T05:19:01.000Z 2025-08-06T05:19:01.000Z Speedup:

1
2
ceph tell 'osd.*' injectargs '--osd-max-backfills 16'
ceph tell 'osd.*' injectargs '--osd-recovery-max-active 4'

Normal:

1
2
ceph tell 'osd.*' injectargs '--osd-max-backfills 1'
ceph tell 'osd.*' injectargs '--osd-recovery-max-active 0'
]]> <p>Speedup:</p> <div class="highlight-container" data-rel="Shell"><figure class="iseeu highlight shell"><table><tr><td class="gutter"><pre>< Ceph - add host https://moki.org.pl/2025/08/05/ceph-add-host/ 2025-08-05T05:29:01.000Z 2025-08-05T05:29:01.000Z List services:

1
cephadm shell ceph orch host add NODE_NAME IP_ADDRESS

Example:

1
cephadm shell ceph orch host add ceph-4 10.10.10.14
]]> <p>List services:</p> <div class="highlight-container" data-rel="Shell"><figure class="iseeu highlight shell"><table><tr><td class="gutter"> Ceph - remove OSD https://moki.org.pl/2025/08/05/ceph-remove-osd/ 2025-08-05T05:29:01.000Z 2025-08-05T05:29:01.000Z List OSD:

1
ceph osd tree

List volumes:

1
cephadm shell -- ceph-volume lvm list

Remove OSD:

1
2
3
4
5
6
7
export ID="11" ; \
ceph osd out $ID ; \
ceph osd down $ID ; \
ceph osd rm $ID ; \
ceph auth del osd.$ID ; \
ceph orch daemon rm osd.$ID --force ; \
ceph osd crush remove osd.$ID

Wipe disk:

1
ceph orch device zap HOST DEVICE --force

Example:

1
ceph orch device zap ceph-node-1 /dev/sde --force
]]> <p>List OSD:</p> <div class="highlight-container" data-rel="Shell"><figure class="iseeu highlight shell"><table><tr><td class="gutter"><pre> Ceph - add all available OSDs https://moki.org.pl/2025/08/05/ceph-add-all-available-osds/ 2025-08-05T05:19:01.000Z 2025-08-05T05:19:01.000Z 
1
ceph orch apply osd --all-available-devices
]]> <div class="highlight-container" data-rel="Shell"><figure class="iseeu highlight shell"><table><tr><td class="gutter"><pre><span class="line Ceph - add new OSD https://moki.org.pl/2025/08/05/ceph-add-new-osd/ 2025-08-05T05:19:01.000Z 2025-08-05T05:19:01.000Z 
1
ceph orch daemon add osd HOSTNAME:DEVICE_PATH

Hosts list:

1
ceph orch host ls

Devices list:

1
ceph orch device ls

Disks list with optional node name:

1
ceph orch device ls [NODE_NAME]

Example: 

1
ceph orch daemon add osd ceph-node-1:/dev/sde
]]> <div class="highlight-container" data-rel="Shell"><figure class="iseeu highlight shell"><table><tr><td class="gutter"><pre><span class="line Ceph - force scrub and deep scrub https://moki.org.pl/2025/08/05/ceph-force-scrub-and-deep-scrub/ 2025-08-05T05:19:01.000Z 2025-08-05T05:19:01.000Z 
1
2
ceph health detail | grep "not scrubbed since" | awk '{ print $2 }' | xargs --no-run-if-empty -n1 ceph pg scrub ; \
ceph health detail | grep "not deep-scrubbed since" | awk '{ print $2 }' | xargs --no-run-if-empty -n1 ceph pg deep-scrub
]]> <div class="highlight-container" data-rel="Shell"><figure class="iseeu highlight shell"><table><tr><td class="gutter"><pre><span class="line Ceph - upgrade https://moki.org.pl/2025/08/05/ceph-upgrade/ 2025-08-05T05:19:01.000Z 2025-08-05T05:19:01.000Z Check for HEALTH_OK:

1
ceph -s

Start upgrade:

1
ceph orch upgrade start --ceph-version 19.2.3

Monitor:

1
watch -n 1 ceph orch upgrade status
]]> <p>Check for <strong>HEALTH_OK</strong>:</p> <div class="highlight-container" data-rel="Shell"><figure class="iseeu highlight shell"><table> Detect disks without reboot https://moki.org.pl/2025/08/04/detect-disks-without-reboot/ 2025-08-04T19:11:19.000Z 2025-08-04T19:11:19.000Z 
1
for host in /sys/class/scsi_host/*; do echo "- - -" | sudo tee $host/scan; ls /dev/sd* ; done
]]> <div class="highlight-container" data-rel="Shell"><figure class="iseeu highlight shell"><table><tr><td class="gutter"><pre><span class="line