update
This commit is contained in:
parent
259e77bc0c
commit
6bb476f64b
14
README.md
14
README.md
|
@ -11,13 +11,13 @@
|
|||
* [caddy_v2](caddy_v2/) - reverse proxy
|
||||
* [vaultwarden](vaultwarden/) - password manager
|
||||
* [bookstack](bookstack/) - notes and documentation
|
||||
* [kopia](kopia_backup/) - backup utility replacing borg
|
||||
* [borg_backup](borg_backup/) - backup utility
|
||||
* [ddclient](ddclient/) - automatic DNS update
|
||||
* [dnsmasq](dnsmasq/) - DNS and DHCP server
|
||||
* [gotify / ntfy / signal](gotify-ntfy-signal/) - instant notifications apps
|
||||
* [homer](homer/) - homepage
|
||||
* [frigate](frigate/) - managing security cameras
|
||||
* [jellyfin](jellyfin/) - video and music streaming
|
||||
* [kopia](kopia_backup/) - backup utility replacing borg
|
||||
* [minecraft](minecraft/) - game server
|
||||
* [meshcrentral](meshcrentral/) - web based remote desktop, like teamviewer or anydesk
|
||||
* [rustdesk](rustdesk/) - remote desktop, like teamviewer or anydesk
|
||||
|
@ -29,6 +29,8 @@
|
|||
* [unifi](unifi/) - management utility for ubiquiti devices
|
||||
* [snipeit](snipeit/) - IT inventory management
|
||||
* [trueNAS scale](trueNASscale/) - network file sharing
|
||||
* [uptime kuma](uptime-kuma/) - uptime alerting tool
|
||||
* [squid](squid/) - anonymize forward proxy
|
||||
* [wireguard](wireguard/) - the one and only VPN to ever consider
|
||||
* [wg-easy](wg-easy/) - wireguard in docker with web gui
|
||||
* [zammad](zammad/) - ticketing system
|
||||
|
@ -171,17 +173,17 @@ now you can ctop anywhere.
|
|||
|
||||
---
|
||||
|
||||
### Sendinblue
|
||||
### Brevo
|
||||
|
||||
Services often need ability to send emails, for notification, registration,
|
||||
password reset and such... Sendinblue is free, offers 300 mails a day
|
||||
and is easy to setup.
|
||||
|
||||
```
|
||||
EMAIL_HOST=smtp-relay.sendinblue.com
|
||||
EMAIL_HOST=smtp-relay.brevo.com
|
||||
EMAIL_PORT=587
|
||||
EMAIL_HOST_USER=whoever_example@gmail.com>
|
||||
EMAIL_HOST_PASSWORD=xcmpwik-c31d9eykwewf2342df2fwfj04-FKLzpHgMjGqP23
|
||||
EMAIL_HOST_USER=whoever_example@gmail.com
|
||||
EMAIL_HOST_PASSWORD=xcmpwik-c31d9eykwef3342df2fwfj04-FKLzpHgMjGqP23
|
||||
EMAIL_USE_TLS=1
|
||||
```
|
||||
|
||||
|
|
|
@ -0,0 +1,85 @@
|
|||
#!/bin/bash
|
||||
|
||||
# this script requires fio bc jq
|
||||
type fio bc jq > /dev/null || exit
|
||||
|
||||
# Directory to test
|
||||
TEST_DIR=$1
|
||||
|
||||
# Parameters for the tests should be representive of the workload you want to simulate
|
||||
BS="1M" # Block size
|
||||
IOENGINE="libaio" # IO engine
|
||||
IODEPTH="16" # IO depth sets how many I/O requests a single job can handle at once
|
||||
DIRECT="1" # Direct IO at 0 is buffered with RAM which may skew results and I/O 1 is unbuffered
|
||||
NUMJOBS="5" # Number of jobs is how many independent I/O streams are being sent to the storage
|
||||
FSYNC="0" # Fsync 0 leaves flushing up to Linux 1 force write commits to disk
|
||||
NUMFILES="5" # Number of files is number of independent I/O threads or processes that FIO will spawn
|
||||
FILESIZE="1G" # File size for the tests, you can use: K M G
|
||||
|
||||
# Check if directory is provided
|
||||
if [ -z "$TEST_DIR" ]; then
|
||||
echo "Usage: $0 [directory]"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Function to perform FIO test and display average output
|
||||
perform_test() {
|
||||
RW_TYPE=$1
|
||||
|
||||
echo "Running $RW_TYPE test with block size $BS, ioengine $IOENGINE, iodepth $IODEPTH, direct $DIRECT, numjobs $NUMJOBS, fsync $FSYNC, using $NUMFILES files of size $FILESIZE on $TEST_DIR"
|
||||
|
||||
# Initialize variables to store cumulative values
|
||||
TOTAL_READ_IOPS=0
|
||||
TOTAL_WRITE_IOPS=0
|
||||
TOTAL_READ_BW=0
|
||||
TOTAL_WRITE_BW=0
|
||||
|
||||
for ((i=1; i<=NUMFILES; i++)); do
|
||||
TEST_FILE="$TEST_DIR/fio_test_file_$i"
|
||||
|
||||
# Running FIO for each file and parsing output
|
||||
OUTPUT=$(fio --name=test_$i \
|
||||
--filename=$TEST_FILE \
|
||||
--rw=$RW_TYPE \
|
||||
--bs=$BS \
|
||||
--ioengine=$IOENGINE \
|
||||
--iodepth=$IODEPTH \
|
||||
--direct=$DIRECT \
|
||||
--numjobs=$NUMJOBS \
|
||||
--fsync=$FSYNC \
|
||||
--size=$FILESIZE \
|
||||
--group_reporting \
|
||||
--output-format=json)
|
||||
|
||||
# Accumulate values
|
||||
TOTAL_READ_IOPS=$(echo $OUTPUT | jq '.jobs[0].read.iops + '"$TOTAL_READ_IOPS")
|
||||
TOTAL_WRITE_IOPS=$(echo $OUTPUT | jq '.jobs[0].write.iops + '"$TOTAL_WRITE_IOPS")
|
||||
TOTAL_READ_BW=$(echo $OUTPUT | jq '(.jobs[0].read.bw / 1024) + '"$TOTAL_READ_BW")
|
||||
TOTAL_WRITE_BW=$(echo $OUTPUT | jq '(.jobs[0].write.bw / 1024) + '"$TOTAL_WRITE_BW")
|
||||
done
|
||||
|
||||
# Calculate averages
|
||||
AVG_READ_IOPS=$(echo "$TOTAL_READ_IOPS / $NUMFILES" | bc -l)
|
||||
AVG_WRITE_IOPS=$(echo "$TOTAL_WRITE_IOPS / $NUMFILES" | bc -l)
|
||||
AVG_READ_BW=$(echo "$TOTAL_READ_BW / $NUMFILES" | bc -l)
|
||||
AVG_WRITE_BW=$(echo "$TOTAL_WRITE_BW / $NUMFILES" | bc -l)
|
||||
|
||||
# Format and print averages, omitting 0 results
|
||||
[ "$(echo "$AVG_READ_IOPS > 0" | bc)" -eq 1 ] && printf "Average Read IOPS: %'.2f\n" $AVG_READ_IOPS
|
||||
[ "$(echo "$AVG_WRITE_IOPS > 0" | bc)" -eq 1 ] && printf "Average Write IOPS: %'.2f\n" $AVG_WRITE_IOPS
|
||||
[ "$(echo "$AVG_READ_BW > 0" | bc)" -eq 1 ] && printf "Average Read Bandwidth (MB/s): %'.2f\n" $AVG_READ_BW
|
||||
[ "$(echo "$AVG_WRITE_BW > 0" | bc)" -eq 1 ] && printf "Average Write Bandwidth (MB/s): %'.2f\n" $AVG_WRITE_BW
|
||||
|
||||
}
|
||||
|
||||
# Run tests
|
||||
perform_test randwrite
|
||||
perform_test randread
|
||||
perform_test write
|
||||
perform_test read
|
||||
perform_test readwrite
|
||||
|
||||
# Clean up
|
||||
for ((i=1; i<=NUMFILES; i++)); do
|
||||
rm "$TEST_DIR/fio_test_file_$i"
|
||||
done
|
|
@ -19,6 +19,8 @@ This repo aims to just have a simple one preset that tells most about the disk.
|
|||
|
||||
# Useful links
|
||||
|
||||
https://www.youtube.com/watch?v=T23uPC6qKeA
|
||||
|
||||
https://www.youtube.com/watch?v=mBhXUYh-76o
|
||||
https://arstechnica.com/gadgets/2020/02/how-fast-are-your-disks-find-out-the-open-source-way-with-fio/
|
||||
https://portal.nutanix.com/page/documents/kbs/details?targetId=kA07V000000LX7xSAG
|
||||
|
|
|
@ -0,0 +1,250 @@
|
|||
# Frigate
|
||||
|
||||
###### guide-by-example
|
||||
|
||||
![logo](https://i.imgur.com/40qhwix.png)
|
||||
|
||||
WORK IN PROGRESS<br>
|
||||
WORK IN PROGRESS<br>
|
||||
WORK IN PROGRESS<br>
|
||||
|
||||
# Purpose & Overview
|
||||
|
||||
|
||||
Managing security cameras - recording, detection, notifications.
|
||||
|
||||
* [Official site](https://frigate.video/)
|
||||
* [Github](https://github.com/blakeblackshear/frigate)
|
||||
|
||||
Frigate is a software NVR - network video recorder.<br>
|
||||
Simple, clean web-based interface with possible integration in to home assistant
|
||||
and its app.
|
||||
|
||||
Frigate offers powerful **AI object detection**, by using OpenCV and Tensorflow.
|
||||
In contrast to cameras of old time which just detect movement,
|
||||
Frigate can recognize if object in view is a cat, a car or a human.
|
||||
|
||||
This detection is cpu heavy and to ease the load,
|
||||
[Google Coral TPU](https://docs.frigate.video/frigate/hardware#google-coral-tpu)
|
||||
is recommended if planning to run multiple cameras with detection.<br>
|
||||
Recently
|
||||
[OpenVINO](https://docs.frigate.video/configuration/detectors/#openvino-detector)
|
||||
has been integrated, which should allow use of igpu of intel 6th+ gen cpus
|
||||
as a detector.
|
||||
|
||||
Open source, written in Python and JavaScript.
|
||||
|
||||
# Files and directory structure
|
||||
|
||||
```
|
||||
/home/
|
||||
└── ~/
|
||||
└── docker/
|
||||
└── frigate/
|
||||
├── 🗁 frigate_storage/
|
||||
├── 🗋 .env
|
||||
├── 🗋 config.yml
|
||||
└── 🗋 docker-compose.yml
|
||||
```
|
||||
|
||||
* `frigate_storage/` - configuration
|
||||
* `transcodes/` - transcoded video storage
|
||||
* `.env` - a file containing environment variables for docker compose
|
||||
* `docker-compose.yml` - a docker compose file, telling docker how to run the containers
|
||||
|
||||
You only need to provide the three files.</br>
|
||||
The directory is created by docker compose on the first run.
|
||||
|
||||
# docker-compose
|
||||
|
||||
* [Official compose file documentation.](https://docs.frigate.video/frigate/installation/#docker)
|
||||
|
||||
This docker compose is based off the official one except few changes.<br>
|
||||
Using bind mounts instead of volumes, moved variables to the `.env` file,
|
||||
commented out privileged mode, increased shm_size,...
|
||||
|
||||
Nothing special going on here,
|
||||
of note is use of `tmpfs` for ram temp storage
|
||||
and [shm_size](https://docs.frigate.video/frigate/installation/#calculating-required-shm-size).
|
||||
|
||||
`docker-compose.yml`
|
||||
```yml
|
||||
services:
|
||||
|
||||
frigate:
|
||||
image: ghcr.io/blakeblackshear/frigate:stable
|
||||
container_name: frigate
|
||||
hostname: frigate
|
||||
restart: unless-stopped
|
||||
env_file: .env
|
||||
# privileged: true
|
||||
shm_size: "256mb"
|
||||
volumes:
|
||||
- /etc/localtime:/etc/localtime:ro
|
||||
- ./config.yml:/config/config.yml
|
||||
- ./frigate_storage:/media/frigate
|
||||
- type: tmpfs # 1GB of memory
|
||||
target: /tmp/cache
|
||||
tmpfs:
|
||||
size: 1000000000
|
||||
ports:
|
||||
- "5000:5000" # Web GUI
|
||||
- "8554:8554" # RTSP feeds
|
||||
- "8555:8555/tcp" # WebRTC over tcp
|
||||
- "8555:8555/udp" # WebRTC over udp
|
||||
|
||||
networks:
|
||||
default:
|
||||
name: $DOCKER_MY_NETWORK
|
||||
external: true
|
||||
```
|
||||
|
||||
`.env`
|
||||
```bash
|
||||
# GENERAL
|
||||
DOCKER_MY_NETWORK=caddy_net
|
||||
TZ=Europe/Bratislava
|
||||
|
||||
# FRIGATE
|
||||
FRIGATE_RTSP_USER: "admin"
|
||||
FRIGATE_RTSP_PASSWORD: "dontlookatmekameras"
|
||||
```
|
||||
|
||||
**All containers must be on the same network**.</br>
|
||||
Which is named in the `.env` file.</br>
|
||||
If one does not exist yet: `docker network create caddy_net`
|
||||
|
||||
# Reverse proxy
|
||||
|
||||
Caddy is used, details
|
||||
[here](https://github.com/DoTheEvo/selfhosted-apps-docker/tree/master/caddy_v2).</br>
|
||||
|
||||
`Caddyfile`
|
||||
```
|
||||
cam.{$MY_DOMAIN} {
|
||||
reverse_proxy frigate:5000
|
||||
}
|
||||
```
|
||||
|
||||
# Configuration - config.yml
|
||||
|
||||
<summary><h3>Terminology</h3></summary>
|
||||
|
||||
* PoE - power over ethernet, camera is powered by the same cat cable that
|
||||
carries data. You want POE(802.3af) or POE+(802.3at),
|
||||
none of the passive poe by mikrotik or ubiquity.
|
||||
* onvif - attempt at industry standard for security cameras, nvr,.. regardless of manufacturer
|
||||
* rtsp - a protocol for streams
|
||||
* ptz - Pan-Tilt-Zoom allows remote movement of a camera
|
||||
* mqtt - messaging protocol to communicate with home assistant
|
||||
|
||||
### Preparation
|
||||
|
||||
Connect camera to your network.
|
||||
|
||||
Find url of your camera streams, either by googling your model,
|
||||
or theres a handy windows utility -
|
||||
[onvif-device-manager](https://sourceforge.net/projects/onvifdm/).
|
||||
Unfortunately all official urls seem dead,
|
||||
[this](https://softradar.com/onvif-device-manager/)
|
||||
worked for me and passed virustotal at the time. There are also comments
|
||||
with some links at its sourceforge page.<br>
|
||||
Camera discovery of onvif-device-manager is almost instant, if the camera requires
|
||||
credentials, set them in the top left corner.<br>
|
||||
In live view there should be stream url displayed. Like: "rtsp://10.0.19.171:554/stream1"
|
||||
|
||||
Ideally your camera has several streams
|
||||
A primary one in full resolution full frame rate for recording,
|
||||
and then secondary one in much smaller resolution and fps for observing.
|
||||
|
||||
### First basic config
|
||||
|
||||
* [Official documentation for config.yml](https://docs.frigate.video/configuration/)
|
||||
|
||||
Example bare config that should shows camera stream once frigate is running.<br>
|
||||
This one has credentails contained in the url - `rtsp://username:password@ip:port/url`
|
||||
|
||||
|
||||
```yml
|
||||
mqtt:
|
||||
enabled: false
|
||||
cameras:
|
||||
C1-Whatever:
|
||||
ffmpeg:
|
||||
inputs:
|
||||
- path: rtsp://{FRIGATE_RTSP_USER}:{FRIGATE_RTSP_PASSWORD}@10.0.19.171:554/stream1
|
||||
```
|
||||
|
||||
All that is there is disabled mqtt since no home assistant yet
|
||||
and just single camera stream that pulls credentails from the `.env` file.
|
||||
|
||||
---
|
||||
|
||||
Now to also record main stream and detect on substream.
|
||||
|
||||
|
||||
```yml
|
||||
mqtt:
|
||||
enabled: false
|
||||
detectors:
|
||||
default_detector_for_all:
|
||||
type: cpu
|
||||
objects:
|
||||
track:
|
||||
- person
|
||||
- cat
|
||||
- dog
|
||||
cameras:
|
||||
K1-Brana:
|
||||
ffmpeg:
|
||||
inputs:
|
||||
- path: rtsp://{FRIGATE_RTSP_USER}:{FRIGATE_RTSP_PASSWORD}@10.0.19.171:554/stream1
|
||||
roles:
|
||||
- record
|
||||
- path: rtsp://{FRIGATE_RTSP_USER}:{FRIGATE_RTSP_PASSWORD}@10.0.19.171:554/stream2
|
||||
roles:
|
||||
- detect
|
||||
detect:
|
||||
width: 640
|
||||
height: 480
|
||||
fps: 5
|
||||
snapshots:
|
||||
enabled: True
|
||||
bounding_box: True
|
||||
record:
|
||||
enabled: True
|
||||
retain:
|
||||
days: 1
|
||||
motion:
|
||||
mask:
|
||||
- 0,480,186,480,174,226,173,0,0,0
|
||||
```
|
||||
|
||||
# First run
|
||||
|
||||
|
||||
|
||||
|
||||
# Specifics of my setup
|
||||
|
||||
|
||||
|
||||
# Troubleshooting
|
||||
|
||||
|
||||
|
||||
|
||||
# Update
|
||||
|
||||
Manual image update:
|
||||
|
||||
- `docker-compose pull`</br>
|
||||
- `docker-compose up -d`</br>
|
||||
- `docker image prune`
|
||||
|
||||
# Backup and restore
|
||||
|
||||
#### Backup
|
||||
|
||||
#### Restore
|
||||
|
|
@ -49,3 +49,8 @@ Works same when pinging from archlinux or pinging from win8.1
|
|||
|
||||
* https://dnsdumpster.com/<br>
|
||||
can check subdomains registered, ideal would be wildcard certificate
|
||||
|
||||
|
||||
OSI Model
|
||||
|
||||
* https://www.youtube.com/watch?v=2iFFRqzX3yE
|
||||
|
|
|
@ -428,6 +428,127 @@ Must **enable logging** for a rule to be visible there.
|
|||
---
|
||||
---
|
||||
|
||||
<details>
|
||||
<summary><h1>Grafana dashboard monitoring</h1></summary>
|
||||
|
||||
![dashboard](https://i.imgur.com/SFd8773.png)
|
||||
|
||||
[bsmithio/OPNsense-Dashboard](https://github.com/bsmithio/OPNsense-Dashboard)
|
||||
seems like amazingly well done thing that everyone would want.. if it was easy.
|
||||
|
||||
Annoying thing is that I invested time and effort in to monitoring my
|
||||
[caddy reverse proxy](https://github.com/DoTheEvo/selfhosted-apps-docker/tree/master/caddy_v2#monitoring)
|
||||
and learning prometheus, loki, promtail,... and literaly the moment I was done
|
||||
I started to think about why not do that for firewall instead of reverse proxy
|
||||
and so I found now bsmithio project that uses completely different stack -
|
||||
mongo, elasticsearch, graylog, influxdb.
|
||||
|
||||
Well, [the documentation](https://github.com/bsmithio/OPNsense-Dashboard/blob/master/configure.md)
|
||||
seems to be excelent so lets try this shit out.
|
||||
|
||||
Though still I learn best by step by step documenting shit as I try it,
|
||||
and make adjustments to my prefernce... so lets try again here.
|
||||
|
||||
```
|
||||
services:
|
||||
|
||||
mongodb:
|
||||
image: mongo:6.0.4
|
||||
container_name: opns-mongo
|
||||
hostname: opns-mongo
|
||||
restart: unless-stopped
|
||||
env_file: .env
|
||||
volumes:
|
||||
- ./mongodb_data:/data/db
|
||||
|
||||
elasticsearch:
|
||||
image: docker.elastic.co/elasticsearch/elasticsearch-oss:7.10.2
|
||||
container_name: opns-elasticsearch
|
||||
hostname: opns-elasticsearch
|
||||
restart: unless-stopped
|
||||
env_file: .env
|
||||
volumes:
|
||||
- ./elasticsearch_data:/usr/share/elasticsearch/data
|
||||
|
||||
graylog:
|
||||
image: graylog/graylog:5.0.2
|
||||
container_name: opns-graylog
|
||||
hostname: opns-graylog
|
||||
restart: unless-stopped
|
||||
env_file: .env
|
||||
volumes:
|
||||
- ./graylog_data:/usr/share/graylog/data
|
||||
depends_on:
|
||||
- mongodb
|
||||
- elasticsearch
|
||||
ports:
|
||||
- "9000:9000" # Graylog web interface and REST API
|
||||
- "1514:1514/udp" # Syslog UDP
|
||||
# - "1514:1514" # Syslog TCP Optional
|
||||
|
||||
influxdb:
|
||||
image: influxdb:2.6.1
|
||||
container_name: opns-influxdb
|
||||
hostname: opns-influxdb
|
||||
restart: unless-stopped
|
||||
env_file: .env
|
||||
ports:
|
||||
- "8086:8086"
|
||||
volumes:
|
||||
- ./influxdb_data:/var/lib/influxdb2
|
||||
|
||||
grafana:
|
||||
image: grafana/grafana:9.4.3
|
||||
container_name: opns-grafana
|
||||
hostname: opns-grafana
|
||||
user: root
|
||||
restart: unless-stopped
|
||||
env_file: .env
|
||||
volumes:
|
||||
- ./grafana_data:/var/lib/grafana
|
||||
depends_on:
|
||||
- influxdb
|
||||
ports:
|
||||
- '3003:3000'
|
||||
|
||||
networks:
|
||||
default:
|
||||
name: $DOCKER_MY_NETWORK
|
||||
external: true
|
||||
```
|
||||
|
||||
```
|
||||
# GENERAL
|
||||
DOCKER_MY_NETWORK=caddy_net
|
||||
TZ=Europe/Bratislava
|
||||
|
||||
# ELASTICSEARCH
|
||||
http.host=0.0.0.0
|
||||
transport.host=localhost
|
||||
network.host=0.0.0.0
|
||||
ES_JAVA_OPTS=-Xms512m -Xmx512m
|
||||
|
||||
# GRAYLOG
|
||||
ROOT_TIMEZONE=Europe/Bratislava
|
||||
GRAYLOG_TIMEZONE=Europe/Bratislava
|
||||
# CHANGE ME (must be at least 16 characters)! This is not your password, this is meant for salting the password below.
|
||||
GRAYLOG_PASSWORD_SECRET=ZicwMzt3NTE4ZzIwM
|
||||
# Username is "admin"
|
||||
# Password is "admin", change this to your own hashed password. 'echo -n "password" | sha256sum'
|
||||
GRAYLOG_ROOT_PASSWORD_SHA2=8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918
|
||||
GRAYLOG_HTTP_EXTERNAL_URI=http://127.0.0.1:9000/
|
||||
|
||||
# GRAFANA
|
||||
GF_SECURITY_ADMIN_USER=opnsense
|
||||
GF_SECURITY_ADMIN_PASSWORD=opnsense
|
||||
# GF_INSTALL_PLUGINS=grafana-worldmap-panel
|
||||
```
|
||||
|
||||
</details>
|
||||
|
||||
---
|
||||
---
|
||||
|
||||
### Extra info and encountered issues
|
||||
|
||||
* Health check - `System: Firmware` Run an audit button, Health
|
||||
|
@ -441,3 +562,6 @@ Must **enable logging** for a rule to be visible there.
|
|||
|
||||
zenarmor that was disabled caused an error notification<br>
|
||||
|
||||
links
|
||||
|
||||
https://homenetworkguy.com/how-to/beginners-guide-to-set-up-home-network-using-opnsense/
|
||||
|
|
|
@ -1,5 +1,7 @@
|
|||
# Port Forwarding
|
||||
|
||||
https://www.reddit.com/r/selfhosted/comments/17tlvs7/i_suppose_im_too_stupid_for_port_forwarding/
|
||||
|
||||
###### guide-by-example
|
||||
|
||||
You want to selfhost stuff.<br>
|
||||
|
|
|
@ -44,6 +44,7 @@ Install and manage software on windows through command line.
|
|||
### Useful
|
||||
|
||||
* search - `scoop search mpv`
|
||||
* `scoop install mpv --global`
|
||||
* search for avaialble pacakges - [scoop.sh](https://scoop.sh/)
|
||||
|
||||
# Choco
|
||||
|
|
Loading…
Reference in New Issue