[{"data":1,"prerenderedAt":442},["ShallowReactive",2],{"i-lucide:menu":3,"i-lucide:arrow-up-right":8,"i-lucide:moon":10,"i-lucide:sun":12,"i-lucide:rss":14,"i-simple-icons:github":16,"i-simple-icons:linkedin":18,"post-\u002F2018\u002Foutage-reports-from-personal-homelab":21,"surround-\u002F2018\u002Foutage-reports-from-personal-homelab":429,"i-lucide:arrow-left":438,"i-lucide:arrow-right":440},{"left":4,"top":4,"width":5,"height":5,"rotate":4,"vFlip":6,"hFlip":6,"body":7},0,24,false,"\u003Cpath fill=\"none\" stroke=\"currentColor\" stroke-linecap=\"round\" stroke-linejoin=\"round\" stroke-width=\"2\" d=\"M4 5h16M4 12h16M4 19h16\"\u002F>",{"left":4,"top":4,"width":5,"height":5,"rotate":4,"vFlip":6,"hFlip":6,"body":9},"\u003Cpath fill=\"none\" stroke=\"currentColor\" stroke-linecap=\"round\" stroke-linejoin=\"round\" stroke-width=\"2\" d=\"M7 7h10v10M7 17L17 7\"\u002F>",{"left":4,"top":4,"width":5,"height":5,"rotate":4,"vFlip":6,"hFlip":6,"body":11},"\u003Cpath fill=\"none\" stroke=\"currentColor\" stroke-linecap=\"round\" stroke-linejoin=\"round\" stroke-width=\"2\" d=\"M20.985 12.486a9 9 0 1 1-9.473-9.472c.405-.022.617.46.402.803a6 6 0 0 0 8.268 8.268c.344-.215.825-.004.803.401\"\u002F>",{"left":4,"top":4,"width":5,"height":5,"rotate":4,"vFlip":6,"hFlip":6,"body":13},"\u003Cg fill=\"none\" stroke=\"currentColor\" stroke-linecap=\"round\" stroke-linejoin=\"round\" stroke-width=\"2\">\u003Ccircle cx=\"12\" cy=\"12\" r=\"4\"\u002F>\u003Cpath d=\"M12 2v2m0 16v2M4.93 4.93l1.41 1.41m11.32 11.32l1.41 1.41M2 12h2m16 0h2M6.34 17.66l-1.41 1.41M19.07 4.93l-1.41 1.41\"\u002F>\u003C\u002Fg>",{"left":4,"top":4,"width":5,"height":5,"rotate":4,"vFlip":6,"hFlip":6,"body":15},"\u003Cg fill=\"none\" stroke=\"currentColor\" stroke-linecap=\"round\" stroke-linejoin=\"round\" stroke-width=\"2\">\u003Cpath d=\"M4 11a9 9 0 0 1 9 9M4 4a16 16 0 0 1 16 16\"\u002F>\u003Ccircle cx=\"5\" cy=\"19\" r=\"1\"\u002F>\u003C\u002Fg>",{"left":4,"top":4,"width":5,"height":5,"rotate":4,"vFlip":6,"hFlip":6,"body":17},"\u003Cpath fill=\"currentColor\" d=\"M12 .297c-6.63 0-12 5.373-12 12c0 5.303 3.438 9.8 8.205 11.385c.6.113.82-.258.82-.577c0-.285-.01-1.04-.015-2.04c-3.338.724-4.042-1.61-4.042-1.61C4.422 18.07 3.633 17.7 3.633 17.7c-1.087-.744.084-.729.084-.729c1.205.084 1.838 1.236 1.838 1.236c1.07 1.835 2.809 1.305 3.495.998c.108-.776.417-1.305.76-1.605c-2.665-.3-5.466-1.332-5.466-5.93c0-1.31.465-2.38 1.235-3.22c-.135-.303-.54-1.523.105-3.176c0 0 1.005-.322 3.3 1.23c.96-.267 1.98-.399 3-.405c1.02.006 2.04.138 3 .405c2.28-1.552 3.285-1.23 3.285-1.23c.645 1.653.24 2.873.12 3.176c.765.84 1.23 1.91 1.23 3.22c0 4.61-2.805 5.625-5.475 5.92c.42.36.81 1.096.81 2.22c0 1.606-.015 2.896-.015 3.286c0 .315.21.69.825.57C20.565 22.092 24 17.592 24 12.297c0-6.627-5.373-12-12-12\"\u002F>",{"left":4,"top":4,"width":5,"height":5,"rotate":4,"vFlip":6,"hFlip":6,"body":19,"hidden":20},"\u003Cpath fill=\"currentColor\" d=\"M20.447 20.452h-3.554v-5.569c0-1.328-.027-3.037-1.852-3.037c-1.853 0-2.136 1.445-2.136 2.939v5.667H9.351V9h3.414v1.561h.046c.477-.9 1.637-1.85 3.37-1.85c3.601 0 4.267 2.37 4.267 5.455v6.286zM5.337 7.433a2.06 2.06 0 0 1-2.063-2.065a2.064 2.064 0 1 1 2.063 2.065m1.782 13.019H3.555V9h3.564zM22.225 0H1.771C.792 0 0 .774 0 1.729v20.542C0 23.227.792 24 1.771 24h20.451C23.2 24 24 23.227 24 22.271V1.729C24 .774 23.2 0 22.222 0z\"\u002F>",true,{"id":22,"title":23,"body":24,"category":416,"comments":417,"date":418,"description":30,"excerpt":417,"extension":419,"image":420,"meta":421,"navigation":20,"path":422,"readingTime":423,"seo":424,"stem":425,"subtitle":426,"tags":417,"wordCount":427,"__hash__":428},"posts\u002F2018\u002Foutage-reports-from-personal-homelab.md","Outage Reports From Personal Homelab",{"type":25,"value":26,"toc":402},"minimark",[27,31,34,37,42,77,84,91,98,104,115,121,124,130,140,143,149,152,157,160,163,169,172,175,179,212,223,233,239,242,245,249,275,278,284,287,290,294,328,337,346,349,352,358,362,365,369,378,381,387,393,396,399],[28,29,30],"p",{},"All times are in local (UTC+2) timezone.",[28,32,33],{},"Consumers, with their ISP-provided all-in-one router\u002Ffirewall\u002Fswitch\u002Faccess point have it easy: plug it in, and hardly anything ever breaks.\nI run a personal homelab from my apartment - a 42U rack with a firewall, switched networking, application servers, UPS - all the good stuff.\nI get more power, but also more moving parts, which means my home IT setup has more resemblance to a corporate environment,\nwith associated problems: more breakage and more support.",[28,35,36],{},"Here are some examples on how things can go wrong.",[38,39,41],"h2",{"id":40},"secondary-wan-unusable-due-to-breached-bandwidth-cap","Secondary WAN Unusable Due To Breached Bandwidth Cap",[43,44,45,57,65,71],"ul",{},[46,47,48,52,53],"li",{},[49,50,51],"strong",{},"Start",": ",[54,55,56],"code",{},"2018-02-18 02:00:40",[46,58,59,52,62],{},[49,60,61],{},"End",[54,63,64],{},"2018-03-01 00:00:00",[46,66,67,70],{},[49,68,69],{},"Root Cause",": Scheduled backups were allowed to run over a fail-over WAN connection, which filled up its data cap and rendered it unusable",[46,72,73,76],{},[49,74,75],{},"Discovered By",": Monitoring",[28,78,79,80,83],{},"During the night, at ",[54,81,82],{},"12:36",", the primary WAN dropped due to excessive packet loss. The WAN connection had been experiencing various degrees of packet loss most of the late evening (11pm...1am, reasons unknown).",[28,85,86],{},[87,88],"img",{"alt":89,"src":90},"Primary WAN down","\u002Fcontent\u002F2018\u002Foutages\u002Fstarman-down.png",[28,92,93,94,97],{},"Shortly after that, a scheduled backup job from a Synology NAS ran. The backup archive was about ",[54,95,96],{},"107 GB"," and was destined to upload to Amazon S3.",[28,99,100],{},[87,101],{"alt":102,"src":103},"Synology backup config","\u002Fcontent\u002F2018\u002Foutages\u002Fbackup-config.png",[28,105,106,107,110,111,114],{},"When the primary WAN (Starman) goes down, a fail-over 4G WAN (Tele2) connection automatically takes over. Even though the primary WAN recovered a while later (",[54,108,109],{},"01:02","), the backup had already started and used the secondary WAN connection, which is bandwidth-capped to ",[54,112,113],{},"30GB",".",[28,116,117],{},[87,118],{"alt":119,"src":120},"Primary WAN recovers","\u002Fcontent\u002F2018\u002Foutages\u002Fstarman-recovery.png",[28,122,123],{},"It is unclear why the backup did not switch back to the primary WAN connection, once it recovered. Leading theory: a persistent TCP connection to\nS3 had already been established over the backup WAN and the firewall \u002F Synology did not want to break the TCP state, so it kept using it.",[28,125,126,127,129],{},"Result - the ",[54,128,113],{}," data cap was eaten up very quickly, the backup failed and the secondary WAN was now effectively offline until the next month,\nwhen the cap reset.",[28,131,132,136],{},[87,133],{"alt":134,"src":135},"Traffic graph of secondary WAN","\u002Fcontent\u002F2018\u002Foutages\u002Ftraffic-graph.png",[87,137],{"alt":138,"src":139},"Data cap full","\u002Fcontent\u002F2018\u002Foutages\u002Fdata-cap.png",[28,141,142],{},"An additional alert was thrown about the failure of the secondary WAN by Prometheus monitoring as soon as the cap was breached\nand the ISP blocked service.",[28,144,145],{},[87,146],{"alt":147,"src":148},"HAProxy back-end down alert","\u002Fcontent\u002F2018\u002Foutages\u002Fhaproxy-alert.png",[28,150,151],{},"The alert reported that a cloud-based HAProxy load-balancer was no longer able to reach my firewall\nvia the secondary WAN connection. This meant that when the primary WAN goes down again, all of my self-hosted webpages will become\nunavailable, too.",[153,154,156],"h3",{"id":155},"mitigation","Mitigation",[28,158,159],{},"The NAS itself is not gateway-aware: it can not detect (without hackery) if the primary or secondary WAN is active and if it should\nproceed with the backups.",[28,161,162],{},"A sensible solution is to apply mitigation from the firewall: only route NAS traffic to WAN through the primary WAN interface (previously: WAN fail-over interface).",[28,164,165],{},[87,166],{"alt":167,"src":168},"NAS firewall rule","\u002Fcontent\u002F2018\u002Foutages\u002Fnas-firewall-rule.png",[28,170,171],{},"The NAS will not reach the Interwebs when the primary WAN is down (a downside), but it will also not eat up data-capped secondary WAN bandwidth again (a win).",[28,173,174],{},"The purpose of the secondary WAN is to keep public webpages accessible during a short primary WAN outage - backups can wait.",[38,176,178],{"id":177},"internet-unusable-due-to-dns-failure","Internet Unusable Due To DNS Failure",[43,180,181,188,195,207],{},[46,182,183,52,185],{},[49,184,51],{},[54,186,187],{},"~2018-02-17 22:00",[46,189,190,52,192],{},[49,191,61],{},[54,193,194],{},"~2018-02-17 22:20",[46,196,197,52,199,206],{},[49,198,69],{},[200,201,205],"a",{"href":202,"rel":203},"https:\u002F\u002Fquad9.net",[204],"nofollow","Quad9"," DNS resolver outage or a network outage to nearest Quad9 server",[46,208,209,211],{},[49,210,75],{},": SO",[28,213,214,215,218,219,222],{},"Incoming customer complaint: \"The Internet is not working\". Investigation revealed that the problematic laptop was in WiFi, had IP\nand was able to ping ",[54,216,217],{},"8.8.8.8",". However, ping to ",[54,220,221],{},"neti.ee"," failed. Further investigation revealed a DNS outage from that laptop\nand from another client device as well. Suspecting a wider DNS outage, investigation moved over to the firewall.",[28,224,225,226,229,230,232],{},"The main firewall was set up to use primary and secondary ",[200,227,205],{"href":202,"rel":228},[204]," DNS servers - it's a DNS service that is both\nfast and also blocks malicious sites at the DNS level, \"for free\". Adding a tertiary DNS (",[54,231,217],{},") to the firewall's pool of\nconfigured DNS servers hotfixed the issue and the customer was happily using the Internet again.",[28,234,235],{},[87,236],{"alt":237,"src":238},"DNS servers","\u002Fcontent\u002F2018\u002Foutages\u002Fdns-servers.png",[28,240,241],{},"Some minutes later, the Quad9 DNS service was working again and the hotfix could be removed.",[28,243,244],{},"Root cause unclear: either an outage at the nearest Quad9 distribution servers or a network failure between me and Quad9.\nNo similar outage has happened to date and Quad9 does not have a service status page (that I could find).",[38,246,248],{"id":247},"momentary-power-spike-from-mains-power","Momentary Power Spike from Mains Power",[43,250,251,258,265,270],{},[46,252,253,52,255],{},[49,254,51],{},[54,256,257],{},"2018-04-17 11:18:03",[46,259,260,52,262],{},[49,261,61],{},[54,263,264],{},"2018-04-17 11:18:10",[46,266,267,269],{},[49,268,69],{},": Unexpected power spike on mains power, over Tallinn city centre, reason unknown",[46,271,272,274],{},[49,273,75],{},": User + Monitoring",[28,276,277],{},"I was working in the office, when, suddenly, the green emergency exit sign on the ceiling flashed to ~300% regular brightness, then back again. Seconds later, I received an alert on my phone that my homelab had gone to battery power - the UPS had taken over. Five seconds later, the UPS was back on mains power.",[28,279,280],{},[87,281],{"alt":282,"src":283},"Kibana logs, showing UPS switching into battery mode","\u002Fcontent\u002F2018\u002Foutages\u002Fups-battery.png",[28,285,286],{},"Theory: an unexpected power spike occured in central Tallinn, my office and home were affected. The spike lasted a second or two.",[28,288,289],{},"The homelab was protected by the UPS.",[38,291,293],{"id":292},"remote-filebeat-log-shipments-failing-due-to-changed-remote-ip-s","Remote Filebeat log shipments failing due to changed remote IP-s",[43,295,296,303,310,318,323],{},[46,297,298,52,300],{},[49,299,51],{},[54,301,302],{},"2019-06-08 19:11:13",[46,304,305,52,307],{},[49,306,61],{},[54,308,309],{},"2019-06-09 16:20:00",[46,311,312,52,315],{},[49,313,314],{},"Detected at",[54,316,317],{},"2019-06-09 10:05:00",[46,319,320,322],{},[49,321,69],{},": Changed remote node IP, which wasn't in a whitelist",[46,324,325,327],{},[49,326,75],{},": User, who needed logs and went looking",[28,329,330,331,336],{},"I am running a ",[200,332,335],{"href":333,"rel":334},"https:\u002F\u002Fwww.digitalocean.com\u002Fproducts\u002Fkubernetes\u002F",[204],"managed Kubernetes cluster"," on DigitalOcean. I run a DaemonSet of Filebeat containers on the nodes, which ship pod and node logs to my on-prem ELK for storage and search. The on-prem firewall has whitelisted the IP-s of DigitalOcean nodes, that are allowed to ship logs to me.",[28,338,339,340,345],{},"As part of the managed service, DigitalOcean performs automatic Kubernetes cluster updates. As details about Intel's MDS vulnerability emerged recently, DigitalOcean ",[200,341,344],{"href":342,"rel":343},"https:\u002F\u002Fblog.digitalocean.com\u002Fmay-2019-intel-vulnerability\u002F",[204],"needed to patch"," their infrastructure. This meant a redeploy of Kubernetes nodes.",[28,347,348],{},"Every time the nodes are redeployed, they are replaced with new droplets, having new IP-s. As you might guess, this breaks IP whitelisting - and I didn't have any detection or automatic remediation in place.",[28,350,351],{},"So, when I went looking for pod logs from ELK to debug a unrelated problem, I found no logs at all - my Kubernetes node IP-s had changed and incoming log shipments were rejected by my firewall.",[28,353,354],{},[87,355],{"alt":356,"src":357},"Firewall blocking logs","\u002Fcontent\u002F2018\u002Foutages\u002Fblocked-log-shipments.png",[153,359,361],{"id":360},"remediation","Remediation",[28,363,364],{},"The solution was easy enough - update the whitelist with new IP-s.",[153,366,368],{"id":367},"monitoring-improvements","Monitoring improvements",[28,370,371,372,377],{},"As this was bound to happen again, I decided to deploy monitoring and alerting to detect this.\n",[200,373,376],{"href":374,"rel":375},"https:\u002F\u002Felastalert.readthedocs.io",[204],"Elastalert"," fit this use-case (alerting from firewall logs) nicely,\nand I'd been meaning to deploy it anyway.",[28,379,380],{},"I deployed it to my Openshift 3 cluster and added the first rule, which will monitor firewall logs for log shipment blocks.",[28,382,383],{},[87,384],{"alt":385,"src":386},"Elastalert rule","\u002Fcontent\u002F2018\u002Foutages\u002Felastalert-rule.png",[28,388,389],{},[87,390],{"alt":391,"src":392},"Elastalert alert","\u002Fcontent\u002F2018\u002Foutages\u002Felastalert-fleep.png",[28,394,395],{},"A better solution would be to hook into DigitalOcean API-s and automatically update the whitelist.",[397,398],"hr",{},[28,400,401],{},"(This post will be updated when more interesting incidents occur)",{"title":403,"searchDepth":404,"depth":404,"links":405},"",2,[406,410,411,412],{"id":40,"depth":404,"text":41,"children":407},[408],{"id":155,"depth":409,"text":156},3,{"id":177,"depth":404,"text":178},{"id":247,"depth":404,"text":248},{"id":292,"depth":404,"text":293,"children":413},[414,415],{"id":360,"depth":409,"text":361},{"id":367,"depth":409,"text":368},"Learning",null,"2018-02-18","md","\u002Fcontent\u002F2018\u002Foutages\u002Fheader.png",{},"\u002F2018\u002Foutage-reports-from-personal-homelab","about 6 minutes",{"title":23,"description":30},"2018\u002Foutage-reports-from-personal-homelab","Running a personal homelab means experiencing service outages. Here are some examples and what was learned from them.",1162,"eNu3leAiM1yQNJspEUyAKgmjEMzptt6YMuvWYgqsaW0",[430,434],{"title":431,"path":432,"stem":433,"children":-1},"\"Shut It!\" - An Arduino Door Alarm","\u002F2018\u002Fshut-it-an-arduino-door-alarm","2018\u002Fshut-it-an-arduino-door-alarm",{"title":435,"path":436,"stem":437,"children":-1},"My Soul is Mine","\u002F2018\u002Fmy-soul-is-mine","2018\u002Fmy-soul-is-mine",{"left":4,"top":4,"width":5,"height":5,"rotate":4,"vFlip":6,"hFlip":6,"body":439},"\u003Cpath fill=\"none\" stroke=\"currentColor\" stroke-linecap=\"round\" stroke-linejoin=\"round\" stroke-width=\"2\" d=\"m12 19l-7-7l7-7m7 7H5\"\u002F>",{"left":4,"top":4,"width":5,"height":5,"rotate":4,"vFlip":6,"hFlip":6,"body":441},"\u003Cpath fill=\"none\" stroke=\"currentColor\" stroke-linecap=\"round\" stroke-linejoin=\"round\" stroke-width=\"2\" d=\"M5 12h14m-7-7l7 7l-7 7\"\u002F>",1790886931246]