[{"data":1,"prerenderedAt":440},["ShallowReactive",2],{"i-lucide:menu":3,"i-lucide:arrow-up-right":8,"i-lucide:moon":10,"i-lucide:sun":12,"i-lucide:rss":14,"i-simple-icons:github":16,"i-simple-icons:linkedin":18,"post-\u002F2016\u002Fchat-room-statistical-analysis":21,"surround-\u002F2016\u002Fchat-room-statistical-analysis":427,"i-lucide:arrow-left":436,"i-lucide:arrow-right":438},{"left":4,"top":4,"width":5,"height":5,"rotate":4,"vFlip":6,"hFlip":6,"body":7},0,24,false,"\u003Cpath fill=\"none\" stroke=\"currentColor\" stroke-linecap=\"round\" stroke-linejoin=\"round\" stroke-width=\"2\" d=\"M4 5h16M4 12h16M4 19h16\"\u002F>",{"left":4,"top":4,"width":5,"height":5,"rotate":4,"vFlip":6,"hFlip":6,"body":9},"\u003Cpath fill=\"none\" stroke=\"currentColor\" stroke-linecap=\"round\" stroke-linejoin=\"round\" stroke-width=\"2\" d=\"M7 7h10v10M7 17L17 7\"\u002F>",{"left":4,"top":4,"width":5,"height":5,"rotate":4,"vFlip":6,"hFlip":6,"body":11},"\u003Cpath fill=\"none\" stroke=\"currentColor\" stroke-linecap=\"round\" stroke-linejoin=\"round\" stroke-width=\"2\" d=\"M20.985 12.486a9 9 0 1 1-9.473-9.472c.405-.022.617.46.402.803a6 6 0 0 0 8.268 8.268c.344-.215.825-.004.803.401\"\u002F>",{"left":4,"top":4,"width":5,"height":5,"rotate":4,"vFlip":6,"hFlip":6,"body":13},"\u003Cg fill=\"none\" stroke=\"currentColor\" stroke-linecap=\"round\" stroke-linejoin=\"round\" stroke-width=\"2\">\u003Ccircle cx=\"12\" cy=\"12\" r=\"4\"\u002F>\u003Cpath d=\"M12 2v2m0 16v2M4.93 4.93l1.41 1.41m11.32 11.32l1.41 1.41M2 12h2m16 0h2M6.34 17.66l-1.41 1.41M19.07 4.93l-1.41 1.41\"\u002F>\u003C\u002Fg>",{"left":4,"top":4,"width":5,"height":5,"rotate":4,"vFlip":6,"hFlip":6,"body":15},"\u003Cg fill=\"none\" stroke=\"currentColor\" stroke-linecap=\"round\" stroke-linejoin=\"round\" stroke-width=\"2\">\u003Cpath d=\"M4 11a9 9 0 0 1 9 9M4 4a16 16 0 0 1 16 16\"\u002F>\u003Ccircle cx=\"5\" cy=\"19\" r=\"1\"\u002F>\u003C\u002Fg>",{"left":4,"top":4,"width":5,"height":5,"rotate":4,"vFlip":6,"hFlip":6,"body":17},"\u003Cpath fill=\"currentColor\" d=\"M12 .297c-6.63 0-12 5.373-12 12c0 5.303 3.438 9.8 8.205 11.385c.6.113.82-.258.82-.577c0-.285-.01-1.04-.015-2.04c-3.338.724-4.042-1.61-4.042-1.61C4.422 18.07 3.633 17.7 3.633 17.7c-1.087-.744.084-.729.084-.729c1.205.084 1.838 1.236 1.838 1.236c1.07 1.835 2.809 1.305 3.495.998c.108-.776.417-1.305.76-1.605c-2.665-.3-5.466-1.332-5.466-5.93c0-1.31.465-2.38 1.235-3.22c-.135-.303-.54-1.523.105-3.176c0 0 1.005-.322 3.3 1.23c.96-.267 1.98-.399 3-.405c1.02.006 2.04.138 3 .405c2.28-1.552 3.285-1.23 3.285-1.23c.645 1.653.24 2.873.12 3.176c.765.84 1.23 1.91 1.23 3.22c0 4.61-2.805 5.625-5.475 5.92c.42.36.81 1.096.81 2.22c0 1.606-.015 2.896-.015 3.286c0 .315.21.69.825.57C20.565 22.092 24 17.592 24 12.297c0-6.627-5.373-12-12-12\"\u002F>",{"left":4,"top":4,"width":5,"height":5,"rotate":4,"vFlip":6,"hFlip":6,"body":19,"hidden":20},"\u003Cpath fill=\"currentColor\" d=\"M20.447 20.452h-3.554v-5.569c0-1.328-.027-3.037-1.852-3.037c-1.853 0-2.136 1.445-2.136 2.939v5.667H9.351V9h3.414v1.561h.046c.477-.9 1.637-1.85 3.37-1.85c3.601 0 4.267 2.37 4.267 5.455v6.286zM5.337 7.433a2.06 2.06 0 0 1-2.063-2.065a2.064 2.064 0 1 1 2.063 2.065m1.782 13.019H3.555V9h3.564zM22.225 0H1.771C.792 0 0 .774 0 1.729v20.542C0 23.227.792 24 1.771 24h20.451C23.2 24 24 23.227 24 22.271V1.729C24 .774 23.2 0 22.222 0z\"\u002F>",true,{"id":22,"title":23,"body":24,"category":413,"comments":414,"date":415,"description":416,"excerpt":414,"extension":417,"image":418,"meta":419,"navigation":20,"path":420,"readingTime":421,"seo":422,"stem":423,"subtitle":424,"tags":414,"wordCount":425,"__hash__":426},"posts\u002F2016\u002Fchat-room-statistical-analysis.md","Chat Room Statistical Analysis",{"type":25,"value":26,"toc":393},"minimark",[27,45,59,62,78,85,90,111,114,137,141,157,160,170,174,177,182,185,191,195,198,204,208,211,217,221,231,237,241,244,250,259,263,266,272,275,279,288,294,297,301,304,310,313,317,320,326,339,343,352,358,361,365,368,374,378,381,384,387],[28,29,30,31,38,39,44],"p",{},"It was the end of December and I wanted to do something interesting with ",[32,33,37],"a",{"href":34,"rel":35},"https:\u002F\u002Flaravel.com",[36],"nofollow","Laravel 5"," over the holidays. I was inspired by ",[32,40,43],{"href":41,"rel":42},"http:\u002F\u002Fblog.jaywayco.co.uk\u002Fchat-room-sentiment-analysis-with-stanford-corenlp-and-elk",[36],"Chat Room Sentiment Analysis With Stanford Corenlp and Elk",", a link that had been sitting in my to-read list since spring. In it, James Conway describes a hackathon project on applying text sentiment analysis algorithms (Stanford NLP) to chat logs. The idea resonated and I decided to do something similar with Fleep.",[28,46,47,52,53,58],{},[32,48,51],{"href":49,"rel":50},"https:\u002F\u002Ffleep.io",[36],"Fleep"," is a relatively new Estonian startup that provides messaging services to individuals and businesses. Think of it as IRC + e-mail in a nice, multi-platform interface. We at ",[32,54,57],{"href":55,"rel":56},"https:\u002F\u002Fbigbank.eu",[36],"Bigbank"," have been using it as the main internal information channel for over a year now; having moved away from Skype after it was sold to Microsoft.",[28,60,61],{},"Long story short, I wanted answers to some specific statistical questions:",[63,64,65,69,72,75],"ul",{},[66,67,68],"li",{},"Do people chat more in Estonian or English?",[66,70,71],{},"What are the most popular chat rooms?",[66,73,74],{},"At what times of day do people usually chat the most?",[66,76,77],{},"...",[28,79,80],{},[81,82],"img",{"alt":83,"src":84},"Summary","\u002Fcontent\u002F2016\u002F02\u002Ffleep-stats\u002Fsummary.png",[86,87,89],"h2",{"id":88},"gathering-the-data","Gathering The Data",[28,91,92,93,98,99,104,105,110],{},"Fleep ",[32,94,97],{"href":95,"rel":96},"https:\u002F\u002Ffleep.io\u002Ffleepapi",[36],"has a publicly available JSON API",", something that Skype never had. Among other things, they don't ",[32,100,103],{"href":101,"rel":102},"https:\u002F\u002Fen.wikipedia.org\u002Fwiki\u002FVendor_lock-in",[36],"vendor lock-in"," you and the user has the option to ",[32,106,109],{"href":107,"rel":108},"https:\u002F\u002Ffleep.io\u002Fblog\u002F2015\u002F09\u002Ffleep-conversation-history-export-is-here",[36],"export his conversation history"," as a JSON file.",[28,112,113],{},"I downloaded my personal conversation history from Fleep and had a ~130MB JSON file with all the data on chat rooms, sent messages and events. However, my goal was to do company-wide statistics, but Fleep data export only includes the conversations a particular user has seen, so I had to accept the fact that whatever numbers I presented, they would not be based on 100% complete data.",[115,116,120,132,133],"blockquote",{"className":117,"lang":119},[118],"twitter-tweet","en",[28,121,123,127,128],{"lang":119,"dir":122},"ltr",[32,124,126],{"href":125},"https:\u002F\u002Ftwitter.com\u002Ffleepio","@fleepio"," Consider splitting data exports into several files. Parsing a 100MB JSON file requires lots of memory. ",[32,129,131],{"href":130},"https:\u002F\u002Ft.co\u002FfEmx7MVlPQ","pic.twitter.com\u002FfEmx7MVlPQ","— Ando David Roots (@SQrooted) ",[32,134,136],{"href":135},"https:\u002F\u002Ftwitter.com\u002FSQrooted\u002Fstatus\u002F680402839865278464","December 25, 2015",[86,138,140],{"id":139},"preparing-to-analyze","Preparing to Analyze",[28,142,143,144,148,149,152,153,156],{},"I created a new Laravel project with a MySQL database and wrote an importer script which would parse the conversation history file and populate ",[145,146,147],"code",{},"conversations",", ",[145,150,151],{},"messages"," and ",[145,154,155],{},"contacts"," tables of the database. The script also filtered out all messages sent before or after 2015 and conversations with fewer than X participants (I wanted only group-chats).",[28,158,159],{},"I realize that MySQL \u002F InnoDB is not the most efficient way of storing and processing the data I had; however, for a one-time project it was good enough. I now had a relational database with all (my) sent and received Fleep messages and meta-data, cleanly split into meaningful rows and columns.",[161,162,167],"pre",{"className":163,"code":165,"language":166},[164],"language-text","+------------------+\n| Tables_in_fleep  |\n+------------------+\n| contacts         |\n| conversations    |\n| emoticons        |\n| messages         |\n| migrations       |\n| post_frequencies |\n| views            |\n| words            |\n+------------------+\n","text",[145,168,165],{"__ignoreMap":169},"",[86,171,173],{"id":172},"analysis","Analysis",[28,175,176],{},"Next, I wrote a series of analysis classes. Each class tried to answer a specific question such as \"How are users distributed geographically?\" by analyzing the dataset I had collected. The results were written into a separate database table, to be displayed later in the UI.",[178,179,181],"h3",{"id":180},"which-conversations-were-the-most-active","Which Conversations Were The Most Active?",[28,183,184],{},"Count the number of messages sent to each conversation, then divide by the number of working days (in a year). Sort in descending order to get the conversations with the most activity per day.",[28,186,187],{},[81,188],{"alt":189,"src":190},"Overall activity","\u002Fcontent\u002F2016\u002F02\u002Ffleep-stats\u002Factivity.png",[178,192,194],{"id":193},"are-people-sending-more-positive-or-negative-emoticons","Are People Sending More Positive Or Negative Emoticons?",[28,196,197],{},"Categorize each Fleep emoticon into \"positive\", \"negative\" or \"neutral\" category. Scan through all sent messages, count the occurrence of each emoticon. Sum the occurrences of positive and negative emoticons to get the percentage of \"good\" vs \"bad\".",[28,199,200],{},[81,201],{"alt":202,"src":203},"Emoticons","\u002Fcontent\u002F2016\u002F02\u002Ffleep-stats\u002Femoticons.png",[178,205,207],{"id":206},"how-has-employee-count-changed-over-the-year","How Has Employee Count Changed Over The Year?",[28,209,210],{},"Take a general conversations that all employees participate in. Monitor \"join\" and \"left\u002Fkicked\" events during the year. Given the current count of participants at the end of the year, we can work backwards to calculate the number of participants on a particular date. Display the result as a line graph.",[28,212,213],{},[81,214],{"alt":215,"src":216},"Employee count","\u002Fcontent\u002F2016\u002F02\u002Ffleep-stats\u002Femployment.png",[178,218,220],{"id":219},"how-are-users-distributed-geographically","How Are Users Distributed Geographically?",[28,222,223,224,148,227,230],{},"Inspect the contacts table. As the company has e-mail addresses based on the employee's country (",[145,225,226],{},".fi",[145,228,229],{},".ee",",...), we can assume the user is from that country. Group the contacts based on the TLD of the e-mail, display the count as a heat-map of countries.",[28,232,233],{},[81,234],{"alt":235,"src":236},"Geographical distribution","\u002Fcontent\u002F2016\u002F02\u002Ffleep-stats\u002Fgeo.png",[178,238,240],{"id":239},"are-people-speaking-more-english-or-estonian","Are People Speaking More English or Estonian?",[28,242,243],{},"The \"official\" language of the company is English yet many of us are Estonian. Are we sticking to the \"official\" language?",[28,245,246],{},[81,247],{"alt":248,"src":249},"Spoken language","\u002Fcontent\u002F2016\u002F02\u002Ffleep-stats\u002Flanguage.png",[28,251,252,253,258],{},"Go over all sent messages, ",[32,254,257],{"href":255,"rel":256},"https:\u002F\u002Fgithub.com\u002Fcrodas\u002FLanguageDetector",[36],"identify the language"," of each message. Display the results as a pie chart.",[178,260,262],{"id":261},"who-writes-the-longest-shortest-messages","Who Writes The Longest \u002F Shortest Messages?",[28,264,265],{},"Kristo likes to write really epic, several-paragraph messages. Is he of the top 5 who do this? Who are his co-conspirators?",[28,267,268],{},[81,269],{"alt":270,"src":271},"Message length","\u002Fcontent\u002F2016\u002F02\u002Ffleep-stats\u002Flength.png",[28,273,274],{},"Go over all contacts. Go over all messages sent by that contact. Sum the number of characters in each of his messages, divide by the number of messages to get an average message length for that contact.",[178,276,278],{"id":277},"which-rooms-have-the-most-positive-negative-emotion","Which Rooms Have The Most Positive \u002F Negative Emotion?",[28,280,281,282,287],{},"Per chat room, go over all the messages sent in that room. Give a \"",[32,283,286],{"href":284,"rel":285},"https:\u002F\u002Fgithub.com\u002Fanroots\u002Fdocker-sentiment",[36],"sentiment score","\" to each message, sum the scores per conversation. The result is the total \"sentiment score\" for that conversation.",[28,289,290],{},[81,291],{"alt":292,"src":293},"Sentiment","\u002Fcontent\u002F2016\u002F02\u002Ffleep-stats\u002Fsentiment.png",[28,295,296],{},"There are natural language processing algorithms that grade text as positive or negative. For example, the sentence \"I like cheese\" might get a score of 1 (positive) while \"I feel miserable and unhappy\" might be graded as -2 (very negative).",[178,298,300],{"id":299},"at-what-times-of-day-do-people-chat","At What Times Of Day Do People Chat?",[28,302,303],{},"Per conversation, inspect the sending time of each message. Group this information by hours.",[28,305,306],{},[81,307],{"alt":308,"src":309},"Time of day","\u002Fcontent\u002F2016\u002F02\u002Ffleep-stats\u002Ftime.png",[28,311,312],{},"This gives insight into related questions like \"at what time to people go to lunch\" or \"when does the work day start for team B\" or \"when do people close their laptops for the day\".",[178,314,316],{"id":315},"which-emoticons-were-the-most-popular","Which Emoticons Were The Most Popular?",[28,318,319],{},"Go over all sent messages, count the occurrence of each emoticon.",[28,321,322],{},[81,323],{"alt":324,"src":325},"Top emoticons","\u002Fcontent\u002F2016\u002F02\u002Ffleep-stats\u002Ftop-emoticons.png",[115,327,329,132,335],{"className":328,"lang":119},[118],[28,330,331,332,334],{"lang":119,"dir":122},"Hey ",[32,333,126],{"href":125},", what are the usage statistics for each of your smileys? Which ones are the most popular?",[32,336,338],{"href":337},"https:\u002F\u002Ftwitter.com\u002FSQrooted\u002Fstatus\u002F691725875713277952","January 25, 2016",[178,340,342],{"id":341},"what-are-the-most-popular-words-in-chats","What Are The Most Popular Words In Chats?",[28,344,345,346,351],{},"Go over all sent messages per conversation, split them into ",[32,347,350],{"href":348,"rel":349},"https:\u002F\u002Fen.wikipedia.org\u002Fwiki\u002FN-gram",[36],"unigrams",", add 1 to the count of that word. Display the top X words in a word cloud with the number of occurrences as weight.",[28,353,354],{},[81,355],{"alt":356,"src":357},"Word cloud","\u002Fcontent\u002F2016\u002F02\u002Ffleep-stats\u002Fwords.png",[28,359,360],{},"This gives insight into the most popular themes in a conversation.",[178,362,364],{"id":363},"who-works-outside-of-normal-hours","Who Works Outside Of Normal Hours?",[28,366,367],{},"Per contact, look at the \"sent\" time of all messages. Count the number of messages sent outside of working hours (before 9, after 6, weekends). You workaholics!",[28,369,370],{},[81,371],{"alt":372,"src":373},"Workaholics","\u002Fcontent\u002F2016\u002F02\u002Ffleep-stats\u002Fworkaholic.png",[86,375,377],{"id":376},"results","Results",[28,379,380],{},"The project took several days to complete, but I'm happy with the result. Fleep is an open platform in the sense that it's really easy to gather data from it and once you have it, you're only limited by your imagination on what you can do with it. I learned more about Laravel 5 and got to play with statistics. The results themselves are meaningful too and provide insight on how we actually use Fleep.",[28,382,383],{},"I presented the results as a single-page interactive web site to the company. Next steps? Rewrite the project (the code is quick and messy) and perhaps set this up as a dynamic service in the net. The statistics could probably be improved, too: for example, when drawing the employee country map, count only the people who've logged in to Fleep in the last month (active users).",[28,385,386],{},"Lots of room for future improvement... but enough fun for this time.",[28,388,389],{},[390,391,392],"em",{},"Some data in the above screenshots is hidden or changed to protect organizational privacy.",{"title":169,"searchDepth":394,"depth":394,"links":395},2,[396,397,398,412],{"id":88,"depth":394,"text":89},{"id":139,"depth":394,"text":140},{"id":172,"depth":394,"text":173,"children":399},[400,402,403,404,405,406,407,408,409,410,411],{"id":180,"depth":401,"text":181},3,{"id":193,"depth":401,"text":194},{"id":206,"depth":401,"text":207},{"id":219,"depth":401,"text":220},{"id":239,"depth":401,"text":240},{"id":261,"depth":401,"text":262},{"id":277,"depth":401,"text":278},{"id":299,"depth":401,"text":300},{"id":315,"depth":401,"text":316},{"id":341,"depth":401,"text":342},{"id":363,"depth":401,"text":364},{"id":376,"depth":394,"text":377},"Projects",null,"2016-02-01","It was the end of December and I wanted to do something interesting with Laravel 5 over the holidays. I was inspired by Chat Room Sentiment Analysis With Stanford Corenlp and Elk, a link that had been sitting in my to-read list since spring. In it, James Conway describes a hackathon project on applying text sentiment analysis algorithms (Stanford NLP) to chat logs. The idea resonated and I decided to do something similar with Fleep.","md","\u002Fcontent\u002F2016\u002F02\u002Ffleep-stats\u002Fheader.jpg",{},"\u002F2016\u002Fchat-room-statistical-analysis","about 7 minutes",{"title":23,"description":416},"2016\u002Fchat-room-statistical-analysis","Using programming and statistics to answer questions about chatting habits",1281,"qrQUms9nJsHUUuJHyvrKq4Qjo7Im7OSfd2pniPwRQiY",[428,432],{"title":429,"path":430,"stem":431,"children":-1},"Wipe Your Phone Before Handing It Over","\u002F2016\u002Fwipe-your-phone-before-handing-it-over","2016\u002Fwipe-your-phone-before-handing-it-over",{"title":433,"path":434,"stem":435,"children":-1},"Beware - I'll Make Your Desk Vibrate","\u002F2016\u002Fbeware-ill-make-your-desk-vibrate","2016\u002Fbeware-ill-make-your-desk-vibrate",{"left":4,"top":4,"width":5,"height":5,"rotate":4,"vFlip":6,"hFlip":6,"body":437},"\u003Cpath fill=\"none\" stroke=\"currentColor\" stroke-linecap=\"round\" stroke-linejoin=\"round\" stroke-width=\"2\" d=\"m12 19l-7-7l7-7m7 7H5\"\u002F>",{"left":4,"top":4,"width":5,"height":5,"rotate":4,"vFlip":6,"hFlip":6,"body":439},"\u003Cpath fill=\"none\" stroke=\"currentColor\" stroke-linecap=\"round\" stroke-linejoin=\"round\" stroke-width=\"2\" d=\"M5 12h14m-7-7l7 7l-7 7\"\u002F>",1790228554919]