ferencd@0: #include "common.h"
ferencd@0: 
ferencd@0: #include <boost/tokenizer.hpp>
ferencd@0: #include <boost/algorithm/string/predicate.hpp>
ferencd@0: #include <boost/algorithm/string.hpp>
ferencd@0: 
ferencd@0: #include <fpaq0.h>
ferencd@0: 
ferencd@0: #include <cstdlib>
ferencd@0: #include <ctime>
ferencd@0: #include <algorithm>
ferencd@0: #include <cassert>
ferencd@0: #include <cstring>
ferencd@0: #include <cmath>
ferencd@0: #include <sstream>
ferencd@0: #include <iterator>
ferencd@0: #include <cctype>
ferencd@0: 
ferencd@0: // Should stay here, do not move before <algorithm> otherwise strange warnings will come
ferencd@0: #include <log.h>
ferencd@0: 
ferencd@0: 
ferencd@0: const std::string platform()
ferencd@0: {
ferencd@0: #ifdef __ANDROID__
ferencd@0:     return HOSTT_ANDROID;
ferencd@0: #elif defined __linux__
ferencd@0:     return HOSTT_LINUX;
ferencd@0: #elif defined _WIN32
ferencd@0:     return HOSTT_WINDOWS;
ferencd@0: #else
ferencd@0:     return HOSTT_UNKNOWN;
ferencd@0: #endif
ferencd@0: }
ferencd@0: 
ferencd@0: namespace unafrog { namespace utils {
ferencd@0: 
ferencd@0: namespace random {
ferencd@0: 
ferencd@0: std::string random_string( size_t length, unafrog::utils::random::random_string_class cls )
ferencd@0: {
ferencd@0:     auto randchar = [cls]() -> char
ferencd@0:     {
ferencd@0:         auto charset = [cls]() -> std::string {
ferencd@0:             switch (cls) {
ferencd@0:             case unafrog::utils::random::random_string_class::RSC_DEC:
ferencd@0:                 return "0123456789";
ferencd@0:             case unafrog::utils::random::random_string_class::RSC_HEX:
ferencd@0:                 return "0123456789abcdef";
ferencd@0:             case unafrog::utils::random::random_string_class::RSC_ASC_DEC:
ferencd@0:                 return "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ";
ferencd@0:             case unafrog::utils::random::random_string_class::RSC_B64:
ferencd@0:                 return "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ+/";
ferencd@0:             case unafrog::utils::random::random_string_class::RSC_FULL:
ferencd@0:                 return "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ|!#$%&/()=?{[]}+\\-_.:,;'*^";
ferencd@0:             }
ferencd@0:             return "10";
ferencd@0:         }();
ferencd@0: 
ferencd@0:         const size_t max_index = (charset.length() - 1);
ferencd@0:         return charset[ rand() % max_index ];
ferencd@0:     };
ferencd@0:     std::string str(length, 0);
ferencd@0:     std::generate_n( str.begin(), length, randchar );
ferencd@0:     return str;
ferencd@0: }
ferencd@0: 
ferencd@0: } //random
ferencd@0: 
ferencd@0: namespace b62 {
ferencd@0: 
ferencd@0: static const char base62_vals[] = "0123456789"
ferencd@0:                                   "abcdefghijklmnopqrstuvwxyz"
ferencd@0:                                   "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
ferencd@0: 
ferencd@0: static const int base62_index[] = {
ferencd@0:        0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,
ferencd@0:        0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,
ferencd@0:        0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,
ferencd@0:        0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,
ferencd@0:     0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09,    0,    0,
ferencd@0:        0,    0,    0,    0,    0, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a,
ferencd@0:     0x2b, 0x2c, 0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36,
ferencd@0:     0x37, 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d,    0,    0,    0,    0,    0,
ferencd@0:        0, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14,
ferencd@0:     0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20,
ferencd@0:     0x21, 0x22, 0x23,
ferencd@0: };
ferencd@0: 
ferencd@0: void strreverse_inplace (char *str)
ferencd@0: {
ferencd@0:     char c;
ferencd@0:     int half;
ferencd@0:     int len;
ferencd@0:     int i;
ferencd@0: 
ferencd@0:     len = strlen(str);
ferencd@0:     half = len >> 1;
ferencd@0:     for (i = 0; i < half; i++) {
ferencd@0:         c = str[i];
ferencd@0:         str[i] = str[len - i - 1];
ferencd@0:         str[len - i - 1] = c;
ferencd@0:     }
ferencd@0: }
ferencd@0: 
ferencd@0: std::string base62_encode (uint64_t val)
ferencd@0: {
ferencd@0:     char str[128] = {0};
ferencd@0:     size_t i = 0, len = 128;
ferencd@0:     int v;
ferencd@0: 
ferencd@0:     assert(str);
ferencd@0:     assert(len > 0);
ferencd@0: 
ferencd@0:     do {
ferencd@0:         if (i + 1 >= len)
ferencd@0:             return "";
ferencd@0:         v = val % 62;
ferencd@0:         str[i++] = base62_vals[v];
ferencd@0:         val = (val - v) / 62;
ferencd@0:     } while (val > 0);
ferencd@0:     str[i] = '\0';
ferencd@0:     strreverse_inplace(str);
ferencd@0: 
ferencd@0:     return std::string(str);
ferencd@0: }
ferencd@0: 
ferencd@0: uint64_t base62_decode (const std::string& str)
ferencd@0: {
ferencd@0:     uint64_t val = 0;
ferencd@0:     char c;
ferencd@0:     int len;
ferencd@0:     int i;
ferencd@0: 
ferencd@0:     len = str.length();
ferencd@0:     for (i = 0; i < len; i++) {
ferencd@0:         c = str[i];
ferencd@0:         if (!isalnum(c)) {
ferencd@0:             return -1;
ferencd@0:         }
ferencd@0:         val += base62_index[(int)c] *(uint64_t) powl(62, len - i - 1);
ferencd@0:     }
ferencd@0: 
ferencd@0:     return val;
ferencd@0: }
ferencd@0: 
ferencd@0: } // b62
ferencd@0: 
ferencd@0: std::string to_upper(const std::string &s)
ferencd@0: {
ferencd@0:     std::string res = s;
ferencd@0:     std::transform(res.begin(), res.end(), res.begin(), ::toupper);
ferencd@0:     return res;
ferencd@0: }
ferencd@0: 
ferencd@0: }} // unafrog::utils
ferencd@0: 
ferencd@0: static std::string replace(std::string subject, const std::string& search, const std::string& replace)
ferencd@0: {
ferencd@0:     size_t pos = 0;
ferencd@0:     while ((pos = subject.find(search, pos)) != std::string::npos)
ferencd@0:     {
ferencd@0:          subject.replace(pos, search.length(), replace);
ferencd@0:          pos += replace.length();
ferencd@0:     }
ferencd@0:     return subject;
ferencd@0: }
ferencd@0: 
ferencd@0: static void encode(std::string& data)
ferencd@0: {
ferencd@0:     std::string buffer;
ferencd@0:     buffer.reserve(data.size());
ferencd@0:     for(size_t pos = 0; pos != data.size(); ++pos)
ferencd@0:     {
ferencd@0:         switch(data[pos])
ferencd@0:         {
ferencd@0:             case '&':  buffer.append("&amp;");       break;
ferencd@0:             case '\"': buffer.append("&quot;");      break;
ferencd@0:             case '\'': buffer.append("&apos;");      break;
ferencd@0:             case '<':  buffer.append("&lt;");        break;
ferencd@0:             case '>':  buffer.append("&gt;");        break;
ferencd@0:             default:   buffer.append(&data[pos], 1); break;
ferencd@0:         }
ferencd@0:     }
ferencd@0:     data.swap(buffer);
ferencd@0: }
ferencd@0: 
ferencd@0: // forward declaration
ferencd@0: static int get_number(const std::string& s, size_t &i);
ferencd@0: 
ferencd@0: std::string unafrog::utils::sanitize_user_input(const std::string &s, bool remove_domains)
ferencd@0: {
ferencd@0:     static auto html_tags = {"a", "abbr", "address", "area", "article",
ferencd@0:                              "aside", "audio", "b", "base", "bdi", "bdo",
ferencd@0:                              "blockquote", "body", "br", "button", "canvas", "caption",
ferencd@0:                              "cite", "code", "col", "colgroup", "data", "datalist",
ferencd@0:                              "dd", "del", "dfn", "div", "dl", "dt", "em", "embed", "fieldset",
ferencd@0:                              "figcaption", "figure", "footer", "form", "h1", "h2", "h3", "h4",
ferencd@0:                              "h5", "h6", "head", "header", "hr", "html", "i", "iframe", "img",
ferencd@0:                              "input", "ins", "kbd", "keygen", "label", "legend", "li", "link",
ferencd@0:                              "main", "map", "mark", "meta", "meter", "nav", "noscript",
ferencd@0:                              "object", "ol", "optgroup", "option", "output", "p", "param",
ferencd@0:                              "pre", "progress", "q", "rb", "rp", "rt", "rtc", "ruby", "s",
ferencd@0:                              "samp", "script", "section", "select", "small", "source", "span",
ferencd@0:                              "strong", "style", "sub", "sup", "table", "tbody", "td",
ferencd@0:                              "template", "textarea", "tfoot", "th", "thead", "time", "title",
ferencd@0:                              "tr", "track", "u", "ul", "var", "video", "wbr", "--#"};
ferencd@0: 
ferencd@0:     static auto window_methods_js = {"window.alert", "window.atob", "window.blur", "window.btoa",
ferencd@0:                                      "window.clearInterval", "window.clearTimeout", "window.close",
ferencd@0:                                      "window.confirm", "window.createPopup", "window.focus", "window.getComputedStyle",
ferencd@0:                                      "window.getSelection", "window.matchMedia", "window.moveBy", "window.moveTo",
ferencd@0:                                      "window.open", "window.print", "window.prompt", "window.resizeBy",
ferencd@0:                                      "window.resizeTo", "window.scroll", "window.scrollBy", "window.scrollTo",
ferencd@0:                                      "window.setInterval", "window.setTimeout", "window.stop",
ferencd@0:                                      // Should be the last one due to the way the history objects are being accessed
ferencd@0:                                      "window." };
ferencd@0: 
ferencd@0:     static auto history_methods_js = {"history.back", "history.forward", "history.go"};
ferencd@0: 
ferencd@0:     static auto location_js = {"location.hash", "location.host", "location.hostname", "location.href", "location.origin",
ferencd@0:                                "location.pathname", "location.port", "location.protocol", "location.search",
ferencd@0:                                "location.assign", "location.reload", "location.replace" };
ferencd@0: 
ferencd@0:     static auto document_js = {"document.activeElement", "document.addEventListener", "document.adoptNode",
ferencd@0:                                "document.anchors", "document.applets", "document.baseURI", "document.body",
ferencd@0:                                "document.close", "document.cookie", "document.createAttribute", "document.createComment",
ferencd@0:                                "document.createDocumentFragment", "document.createElement", "document.createTextNode",
ferencd@0:                                "document.doctype", "document.documentElement", "document.documentMode",
ferencd@0:                                "document.documentURI", "document.domain", "document.domConfig", "document.embeds",
ferencd@0:                                "document.forms", "document.getElementById", "document.getElementsByClassName",
ferencd@0:                                "document.getElementsByName", "document.getElementsByTagName", "document.hasFocus",
ferencd@0:                                "document.head", "document.images", "document.implementation", "document.importNode",
ferencd@0:                                "document.inputEncoding", "document.lastModified", "document.links",
ferencd@0:                                "document.normalize", "document.normalizeDocument", "document.open", "document.querySelector",
ferencd@0:                                "document.querySelectorAll", "document.readyState", "document.referrer",
ferencd@0:                                "document.removeEventListener", "document.renameNode", "document.scripts",
ferencd@0:                                "document.strictErrorChecking", "document.title", "document.URL", "document.write",
ferencd@0:                                "document.writeln"};
ferencd@0: 
ferencd@0:     static auto js_events = {"onclick", "oncontextmenu", "ondblclick", "onmousedown", "onmouseenter", "onmouseleave",
ferencd@0:                           "onmousemove", "onmouseover", "onmouseout", "onmouseup", "onkeydown", "onkeypress", "onkeyup",
ferencd@0:                           "onabort", "onbeforeunload", "onerror", "onhashchange", "onload", "onpageshow", "onpagehide",
ferencd@0:                           "onresize", "onscroll", "onunload", "onblur", "onchange", "onfocus", "onfocusin", "onfocusout",
ferencd@0:                           "oninput", "oninvalid", "onreset", "onsearch", "onselect", "onsubmit", "ondrag", "ondragend",
ferencd@0:                           "ondragenter", "ondragleave", "ondragover", "ondragstart", "ondrop", "oncopy", "oncut",
ferencd@0:                           "onpaste", "onafterprint", "onbeforeprint", "onabort", "oncanplay", "oncanplaythrough",
ferencd@0:                           "ondurationchange", "onemptied", "onended", "onerror", "onloadeddata", "onloadedmetadata",
ferencd@0:                           "onloadstart", "onpause", "onplay", "onplaying", "onprogress", "onratechange", "onseeked",
ferencd@0:                           "onseeking", "onstalled", "onsuspend", "ontimeupdate", "onvolumechange", "onwaiting",
ferencd@0:                           "animationend", "animationiteration", "animationstart", "transitionend", "onerror",
ferencd@0:                           "onmessage", "onopen", "onmessage", "onmousewheel", "ononline", "onoffline", "onpopstate",
ferencd@0:                           "onshow", "onstorage", "ontoggle", "onwheel", "ontouchcancel", "ontouchend", "ontouchmove",
ferencd@0:                           "ontouchstart", "cancelable", "currentTarget", "defaultPrevented", "eventPhase",
ferencd@0:                           "isTrusted", "timeStamp", "preventDefault", "stopImmediatePropagation", "stopPropagation",
ferencd@0:                           "altKey", "clientX", "clientY", "ctrlKey", "metaKey", "pageX", "pageY", "relatedTarget",
ferencd@0:                           "screenX", "screenY", "shiftKey", "altKey", "ctrlKey", "charCode", "keyCode",
ferencd@0:                           "metaKey", "shiftKey", "newURL", "oldURL", "relatedTarget", "animationName", "elapsedTime",
ferencd@0:                           "propertyName", "elapsedTime", "deltaX", "deltaY", "deltaZ", "deltaMode" };
ferencd@0: 
ferencd@0:     static auto js_globals = {"decodeURI","decodeURIComponent", "encodeURI", "encodeURIComponent",
ferencd@0:                               "eval", "isFinite", "isNaN", "Number", "parseFloat", "parseInt", "String", "unescape" };
ferencd@0: 
ferencd@0:     static auto js_navigator = {"appCodeName", "appName", "appVersion", "cookieEnabled",
ferencd@0:                                 "geolocation", "onLine", "userAgent" };
ferencd@0: 
ferencd@0:     static auto toplevel_domains = {".academy",".accountant",".accountants",".cloud",".active",".actor",".adult",".aero",".agency",".airforce",
ferencd@0:                 ".apartments",".app",".archi",".army",".associates",".attorney",".auction",".audio",".autos",".band",".bar",
ferencd@0:                 ".bargains",".beer",".best",".bid",".bike",".bingo",".bio",".biz",".black",".blackfriday",".blog",".blue",
ferencd@0:                 ".boo",".boutique",".build",".builders",".business",".buzz",".cab",".camera",".camp",".cancerresearch",
ferencd@0:                 ".capital",".cards",".care",".career",".careers",".cash",".casino",".catering",".center",".ceo",".channel",
ferencd@0:                 ".chat",".cheap",".christmas",".church",".city",".claims",".cleaning",".click",".clinic",".clothing",
ferencd@0:                 ".club",".coach",".codes",".coffee",".college",".community",".company",".computer",".condos",".construction",
ferencd@0:                 ".consulting",".contractors",".cooking",".cool",".coop",".country",".coupons",".credit",".creditcard",
ferencd@0:                 ".cricket",".cruises",".dad",".dance",".date",".dating",".day",".deals",".degree",".delivery",".democrat",
ferencd@0:                 ".dental",".dentist",".design",".diamonds",".diet",".digital",".direct",".directory",".discount",".dog",
ferencd@0:                 ".domains",".download",".eat",".education",".email",".energy",".engineer",".engineering",".equipment",
ferencd@0:                 ".esq",".estate",".events",".exchange",".expert",".exposed",".express",".fail",".faith",".family",
ferencd@0:                 ".fans",".farm",".fashion",".pid",".finance",".financial",".fish",".fishing",".fit",".fitness",".flights",
ferencd@0:                 ".florist",".flowers",".fly",".foo",".football",".forsale",".foundation",".fund",".furniture",".fyi",
ferencd@0:                 ".gallery",".garden",".gift",".gifts",".gives",".glass",".global",".gold",".golf",".gop",".graphics",
ferencd@0:                 ".green",".gripe",".guide",".guitars",".guru",".healthcare",".help",".here",".hiphop",".hiv",".hockey",
ferencd@0:                 ".holdings",".holiday",".homes",".horse",".host",".hosting",".house",".how",".info",".ing",".ink",
ferencd@0:                 ".institute[59]",".insure",".international",".investments",".jewelry",".jobs",".kim",".kitchen",".land",
ferencd@0:                 ".lawyer",".lease",".legal",".lgbt",".life",".lighting",".limited",".limo",".link",".loan",".loans",
ferencd@0:                 ".lol",".lotto",".love",".luxe",".luxury",".management",".market",".marketing",".markets",".mba",".media",
ferencd@0:                 ".meet",".meme",".memorial",".men",".menu",".mobi",".moe",".money",".mortgage",".motorcycles",".mov",
ferencd@0:                 ".movie",".museum",".name",".navy",".network",".new",".news",".ngo",".ninja",".one",".ong",".onl",
ferencd@0:                 ".online",".ooo",".organic",".partners",".parts",".party",".pharmacy",".photo",".photography",".photos",
ferencd@0:                 ".physio",".pics",".pictures",".feedback",".pink",".pizza",".place",".plumbing",".plus",".poker",".porn",
ferencd@0:                 ".post",".press",".pro",".productions",".prof",".properties",".property",".qpon",".racing",".recipes",
ferencd@0:                 ".red",".rehab",".ren",".rent",".rentals",".repair",".report",".republican",".rest",".review",".reviews",
ferencd@0:                 ".rich",".rip",".rocks",".rodeo",".rsvp",".run",".sale",".school",".science",".services",".sex",".sexy",
ferencd@0:                 ".shoes",".show",".singles",".site",".soccer",".social",".software",".solar",".solutions",".space",
ferencd@0:                 ".studio",".style",".sucks",".supplies",".supply",".support",".surf",".surgery",".systems",".tattoo",
ferencd@0:                 ".tax",".taxi",".team",".tech",".technology",".tel",".tennis",".theater",".tips",".tires",".today",
ferencd@0:                 ".tools",".top",".tours",".town",".toys",".trade",".training",".travel",".university",".vacations",
ferencd@0:                 ".vet",".video",".villas",".vision",".vodka",".vote",".voting",".voyage",".wang",".watch",".webcam",
ferencd@0:                 ".website",".wed",".wedding",".whoswho",".wiki",".win",".wine",".work",".works",".world",".wtf",
ferencd@0:                 ".xxx",".xyz",".yoga",".zone",".maison",".abogado",".gratis",".futbol",".juegos",".soy",".tienda",
ferencd@0:                 ".uno",".viajes",".haus",".immobilien",".jetzt",".kaufen",".reise",".reisen",".schule",".versicherung",
ferencd@0:                 ".desi",".shiksha",".casa",".cafe",".immo",".moda",".voto",".bar",".bank",".coop",".enterprises",
ferencd@0:                 ".industries",".institute",".ltda",".pub",".realtor",".reit",".rest",".restaurant",".sarl",".ventures",
ferencd@0:                 ".capetown",".durban",".joburg",".asia",".krd",".nagoya",".okinawa",".ryukyu",".taipei",".tatar",".tokyo",
ferencd@0:                 ".yokohama",".alsace",".amsterdam",".barcelona",".bayern",".berlin",".brussels",".budapest",".bzh",
ferencd@0:                 ".cat",".cologne",".corsica",".cymru",".eus",".frl",".gal",".gent",".hamburg",".irish",".koeln",".london",
ferencd@0:                 ".madrid",".moscow",".nrw",".paris",".ruhr",".saarland",".scot",".tirol",".vlaanderen",".wales",".wien",
ferencd@0:                 ".zuerich",".miami",".nyc",".quebec",".vegas",".kiwi",".melbourne",".sydney",".lat",".rio",".allfinanz",
ferencd@0:                 ".android",".aquarelle",".axa",".barclays",".barclaycard",".bloomberg",".bmw",".bnl",".bnpparibas",".cal",
ferencd@0:                 ".caravan",".cern",".chrome",".citic",".crs",".cuisinella",".dnp",".dvag",".emerck",".everbank",".firmdale",
ferencd@0:                 ".flsmidth",".frogans",".gbiz",".gle",".globo",".gmail",".gmo",".gmx",".google",".hsbc",".ibm",".kred",
ferencd@0:                 ".lacaixa",".latrobe",".lds",".mango",".mini",".monash",".mormon",".neustar",".nexus",".nhk",".nico",".nra",
ferencd@0:                 ".otsuka",".ovh",".piaget",".pohl",".praxi",".prod",".pwc",".sandvikcoromant",".sca",".scb",".schmidt",".sohu",
ferencd@0:                 ".spiegel",".suzuki",".tui",".uol",".williamhill",".wme",".wtc",".yandex",".youtube",".com",".org",".net",
ferencd@0:                 ".int",".edu",".gov",".mil",".arpa",".ac",".ad",".ae",".af",".ag",".ai",".al",".am",".an",".ao",".aq",".ar",
ferencd@0:                 ".as",".at",".au",".aw",".ax",".az",".ba",".bb",".bd",".be",".bf",".bg",".bh",".bi",".bj",".bm",".bn",".bo",
ferencd@0:                 ".bq",".br",".bs",".bt",".bv",".bw",".by",".bz",".ca",".cc",".cd",".cf",".cg",".ch",".ci",".ck",".cl",".cm",
ferencd@0:                 ".cn",".co",".cr",".cu",".cv",".cw",".cx",".cy",".cz",".de",".dj",".dk",".dm",".do",".dz",".ec",".ee",".eg",
ferencd@0:                 ".eh",".er",".es",".et",".eu",".fi",".fj",".fk",".fm",".fo",".fr",".ga",".gb",".gd",".ge",".gf",".gg",".gh",
ferencd@0:                 ".gi",".gl",".gm",".gn",".gp",".gq",".gr",".gs",".gt",".gu",".gw",".gy",".hk",".hm",".hn",".hr",".ht",".hu",
ferencd@0:                 ".id",".ie",".il",".im",".in",".io",".iq",".ir",".is",".it",".je",".jm",".jo",".jp",".ke",".kg",".kh",".ki",
ferencd@0:                 ".km",".kn",".kp",".kr",".kw",".ky",".kz",".la",".lb",".lc",".li",".lk",".lr",".ls",".lt",".lu",".lv",".ly",
ferencd@0:                 ".ma",".mc",".md",".me",".mg",".mh",".mk",".ml",".mm",".mn",".mo",".mp",".mq",".mr",".ms",".mt",".mu",".mv",
ferencd@0:                 ".mw",".mx",".my",".mz",".na",".nc",".ne",".nf",".ng",".ni",".nl",".no",".np",".nr",".nu",".nz",".om",".pa",
ferencd@0:                 ".pe",".pf",".pg",".ph",".pk",".pl",".pm",".pn",".pr",".ps",".pt",".pw",".py",".qa",".re",".ro",".rs",".ru",
ferencd@0:                 ".rw",".sa",".sb",".sc",".sd",".se",".sg",".authenticator.cloudy.sh",".si",".sj",".sk",".sl",".sm",".sn",".so",".sr",".ss",".st",
ferencd@0:                 ".su",".sv",".sx",".sy",".sz",".tc",".td",".tf",".tg",".th",".tj",".tk",".tl",".tm",".tn",".to",".tp",".tr",
ferencd@0:                 ".tt",".tv",".tw",".tz",".ua",".ug",".uk",".us",".uy",".uz",".va",".vc",".ve",".vg",".vi",".vn",".vu",".wf",
ferencd@0:                 ".ws",".ye",".yt",".za",".zm",".zw"};
ferencd@0: 
ferencd@0: 
ferencd@0:     static auto with_domains = {window_methods_js, history_methods_js, location_js, document_js, js_events, js_globals, js_navigator,toplevel_domains};
ferencd@0:     static auto without_domains = {window_methods_js, history_methods_js, location_js, document_js, js_events, js_globals, js_navigator};
ferencd@0:     static auto containers = remove_domains ? with_domains : without_domains;
ferencd@0: 
ferencd@0:     std::string result = s;
ferencd@0: 
ferencd@0:     // First run: HTML tags
ferencd@0:     for(auto tag : html_tags)
ferencd@0:     {
ferencd@0: 
ferencd@0:         // Ddi we parse out all the garbage?
ferencd@0:         if(result.empty())
ferencd@0:         {
ferencd@0:             break;
ferencd@0:         }
ferencd@0: 
ferencd@0:         // Zero: Standalone tags
ferencd@0:         std::string open_tag = std::string("<") + tag + std::string(">");
ferencd@0:         result = replace(result, open_tag, "");
ferencd@0:         std::string close_tag = std::string("</") + tag + std::string(">");
ferencd@0:         result = replace(result, close_tag, "");
ferencd@0: 
ferencd@0:         // One: Tags which might have parameters, such as: <script language = "blabl">
ferencd@0:         // Involves parsing out the entire tag
ferencd@0: 
ferencd@0:         std::string endless_tag = std::string("<") + tag;
ferencd@0:         std::size_t etpos = result.find(endless_tag);
ferencd@0:         if( etpos != std::string::npos)
ferencd@0:         {
ferencd@0:             std::string tmps = result.substr(etpos);
ferencd@0:             size_t i = 0;   // i will point to the first character after the endless tag
ferencd@0:             while(i < tmps.length())
ferencd@0:             {
ferencd@0:                 // skip stuff on double quotes
ferencd@0:                 if(tmps[i] == '\"')
ferencd@0:                 {
ferencd@0:                     i ++;
ferencd@0:                     while(i < tmps.length())
ferencd@0:                     {
ferencd@0:                         // skip the escaped double quotes
ferencd@0:                         if(tmps[i] == '\\') i++;
ferencd@0:                         if(i < tmps.length() && tmps[i] == '\"') i++;
ferencd@0: 
ferencd@0:                         // advance
ferencd@0:                         if(i < tmps.length() ) i++;
ferencd@0:                     }
ferencd@0:                 }
ferencd@0: 
ferencd@0:                 if( i < tmps.length() )
ferencd@0:                 {
ferencd@0:                     if(tmps[i] == '>')
ferencd@0:                     {
ferencd@0:                         break;
ferencd@0:                     }
ferencd@0:                     i++;
ferencd@0:                 }
ferencd@0:             }
ferencd@0: 
ferencd@0:             i++;
ferencd@0: 
ferencd@0:             if( i<tmps.length())
ferencd@0:             {
ferencd@0:                 result = tmps.substr(i);
ferencd@0:             }
ferencd@0:             else
ferencd@0:             {
ferencd@0:                 result.clear();
ferencd@0:             }
ferencd@0:         }
ferencd@0:     }
ferencd@0: 
ferencd@0:     // Second run: various javascript stuff
ferencd@0:     for(auto container : containers)
ferencd@0:     {
ferencd@0:         for(auto tag : container)
ferencd@0:         {
ferencd@0:             result = replace(result, tag, "");
ferencd@0:         }
ferencd@0:     }
ferencd@0: 
ferencd@0: again:
ferencd@0:     size_t i = 0;
ferencd@0:     while(i < result.length())
ferencd@0:     {
ferencd@0:         // the first part
ferencd@0:         if(isdigit(result[i]))
ferencd@0:         {
ferencd@0:             size_t save_i = i;
ferencd@0:             int p1 = get_number(result, i);
ferencd@0:             if(p1 == 2)
ferencd@0:             {
ferencd@0:                 // skip the '.'
ferencd@0:                 i++;
ferencd@0:                 // get the next number
ferencd@0:                 int p2 = get_number(result, i);
ferencd@0:                 if(p2 == 2)
ferencd@0:                 {
ferencd@0:                     i++;
ferencd@0:                     int p3 = get_number(result, i);
ferencd@0:                     if(p3 == 2)
ferencd@0:                     {
ferencd@0:                         // skip '.'
ferencd@0:                         i++;
ferencd@0:                         int p4 = get_number(result, i);
ferencd@0:                         if(p4 == 1)
ferencd@0:                         {
ferencd@0:                             // now remove from result the section between save_i and i
ferencd@0:                             result = result.substr(0, save_i) + result.substr(i);
ferencd@0:                             goto again;
ferencd@0:                         }
ferencd@0:                     }
ferencd@0:                 }
ferencd@0:             }
ferencd@0:             else
ferencd@0:             {
ferencd@0:                 continue;
ferencd@0:             }
ferencd@0: 
ferencd@0:         }
ferencd@0:         else
ferencd@0:         {
ferencd@0:             // move to the next character
ferencd@0:             i++;
ferencd@0:         }
ferencd@0:     }
ferencd@0: 
ferencd@0:     // Third run: Encode the remaining XML/HTML artifacts into web-safe data
ferencd@0:     encode(result);
ferencd@0:     return result;
ferencd@0: }
ferencd@0: 
ferencd@0: // returns: 0 - not a valid IP number, 1 - is not followed by '.', 2 - valid IP number, followed by '.'
ferencd@0: static int get_number(const std::string& s, size_t& i)
ferencd@0: {
ferencd@0:     std::string ip1 = "";
ferencd@0:     while(i < s.length() && isdigit(s[i]))
ferencd@0:     {
ferencd@0:         ip1 += s[i++];
ferencd@0:     }
ferencd@0: 
ferencd@0:     // is this a valid IP part?
ferencd@0:     int nip1 = atoi(ip1.c_str());
ferencd@0:     if(nip1 > 255)
ferencd@0:     {
ferencd@0:         return 0;
ferencd@0:     }
ferencd@0: 
ferencd@0:     if(s[i] == '.')
ferencd@0:     {
ferencd@0:         return 2;
ferencd@0:     }
ferencd@0:     return 1;
ferencd@0: }
ferencd@0: 
ferencd@0: static bool case_insensitive_eq(const char &lhs, const char &rhs)
ferencd@0: {
ferencd@0:     return std::toupper(lhs) == std::toupper(rhs);
ferencd@0: }
ferencd@0: 
ferencd@0: static bool case_insensitive_string_eq(const std::string &lhs, const std::string &rhs)
ferencd@0: {
ferencd@0:     return std::equal(lhs.begin(),
ferencd@0:                       lhs.end(),
ferencd@0:                       rhs.begin(),
ferencd@0:                       case_insensitive_eq);
ferencd@0: }
ferencd@0: 
ferencd@0: case_insensitive_str_eq::case_insensitive_str_eq(std::string key) : key_(std::move(key)) {}
ferencd@0: 
ferencd@0: bool case_insensitive_str_eq::operator()(const std::string &item) const
ferencd@0: {
ferencd@0:     if(item.length() != key_.length())
ferencd@0:     {
ferencd@0:         return false;
ferencd@0:     }
ferencd@0:     return case_insensitive_string_eq(item, key_);
ferencd@0: }
ferencd@0: 
ferencd@0: std::string unafrog::server()
ferencd@0: {
ferencd@0:    return "http://localhost:8838";
ferencd@0: }
ferencd@0: 
ferencd@0: std::string unafrog::utils::consume(std::string& s, int c)
ferencd@0: {
ferencd@0:     try
ferencd@0:     {
ferencd@0:         std::string v = s.substr(0, c);
ferencd@0:         s.erase(0, c);
ferencd@0:         return v;
ferencd@0:     }
ferencd@0:     catch (std::exception& ex)
ferencd@0:     {
ferencd@0:         log_err() << "Cannot consume" << c << "characters from" << s << ": " << ex.what();
ferencd@0:         return "";
ferencd@0:     }
ferencd@0:     catch(...)
ferencd@0:     {
ferencd@0:         std::stringstream ss;
ferencd@0:         ss << "Cannot consume" << c << "characters from" << s;
ferencd@0:         log_err() << ss.str();
ferencd@0:         return "";
ferencd@0:     }
ferencd@0: }
ferencd@0: 
ferencd@0: std::vector<uint8_t> unafrog::utils::hex_string_to_vector(const std::string &s)
ferencd@0: {
ferencd@0:     std::vector<uint8_t> out;
ferencd@0:     for (unsigned int i = 0; i < s.length(); i += 2)
ferencd@0:     {
ferencd@0:       std::string byteString = s.substr(i, 2);
ferencd@0:       uint8_t byte = static_cast<uint8_t>(strtol(byteString.c_str(), NULL, 16));
ferencd@0:       out.push_back(byte);
ferencd@0:     }
ferencd@0:     return out;
ferencd@0: }
ferencd@0: 
ferencd@0: std::string unafrog::utils::string_to_hex(const std::string& input)
ferencd@0: {
ferencd@0:     static const char* const lut = "0123456789abcdef";
ferencd@0:     size_t len = input.length();
ferencd@0: 
ferencd@0:     std::string output;
ferencd@0:     output.reserve(2 * len);
ferencd@0:     for (size_t i = 0; i < len; ++i)
ferencd@0:     {
ferencd@0:         const unsigned char c = input[i];
ferencd@0:         output.push_back(lut[c >> 4]);
ferencd@0:         output.push_back(lut[c & 15]);
ferencd@0:     }
ferencd@0:     return output;
ferencd@0: }
ferencd@0: 
ferencd@0: std::string unafrog::utils::hex_to_string(const std::string& input)
ferencd@0: {
ferencd@0:     static const char* const lut = "0123456789abcdef";
ferencd@0:     size_t len = input.length();
ferencd@0:     if (len & 1)
ferencd@0:     {
ferencd@0:         log_err() << "odd length for " + input;
ferencd@0:         throw std::exception();
ferencd@0:     }
ferencd@0: 
ferencd@0:     std::string output;
ferencd@0:     output.reserve(len / 2);
ferencd@0:     for (size_t i = 0; i < len; i += 2)
ferencd@0:     {
ferencd@0:         char a = input[i];
ferencd@0:         const char* p = std::lower_bound(lut, lut + 16, a);
ferencd@0:         if (*p != a)
ferencd@0:         {
ferencd@0:             log_err() << "not a hex digit";
ferencd@0:             throw std::exception();
ferencd@0:         }
ferencd@0: 
ferencd@0:         char b = input[i + 1];
ferencd@0:         const char* q = std::lower_bound(lut, lut + 16, b);
ferencd@0:         if (*q != b)
ferencd@0:         {
ferencd@0:             log_err() << "not a hex digit in: " + input;
ferencd@0:             throw std::exception();
ferencd@0:         }
ferencd@0: 
ferencd@0:         output.push_back(((p - lut) << 4) | (q - lut));
ferencd@0:     }
ferencd@0:     return output;
ferencd@0: }
ferencd@0: 
ferencd@0: std::string unafrog::utils::grow(const std::string &s, std::size_t required_length)
ferencd@0: {
ferencd@0:     std::string result = s;
ferencd@0:     while(result.length() < required_length)
ferencd@0:     {
ferencd@0:         result = "0" + result;
ferencd@0:     }
ferencd@0:     return result;
ferencd@0: }
ferencd@0: 
ferencd@0: std::string join(const std::vector<std::string> &vec, const char *delim)
ferencd@0: {
ferencd@0:     std::stringstream res;
ferencd@0:     std::copy(vec.begin(), vec.end(), std::ostream_iterator<std::string>(res, delim));
ferencd@0:     return res.str();
ferencd@0: }
ferencd@0: 
ferencd@0: #if !defined __ANDROID__
ferencd@0: std::vector<std::string> split(const std::string &s, const char* delim)
ferencd@0: {
ferencd@0:     std::vector<std::string> result;
ferencd@0:     boost::char_separator<char> sep(delim);
ferencd@0:     boost::tokenizer<boost::char_separator<char>> tok(s, sep);
ferencd@0: 
ferencd@0:     for(boost::tokenizer<boost::char_separator<char>>::iterator beg=tok.begin(); beg!=tok.end(); ++beg)
ferencd@0:     {
ferencd@0:         result.push_back(*beg);
ferencd@0:     }
ferencd@0: 
ferencd@0:     return result;
ferencd@0: }
ferencd@0: #endif
ferencd@0: 
ferencd@0: unsigned internalServerError()
ferencd@0: {
ferencd@0:     return 500; // HTTP_INTERNAL_SERVER_ERROR;
ferencd@0: }
ferencd@0: 
ferencd@0: std::string remove_duplicates(std::string s, char to_remove)
ferencd@0: {
ferencd@0:     std::string result;
ferencd@0:     auto i = s.begin();
ferencd@0:     while(i != s.end())
ferencd@0:     {
ferencd@0:         auto pv = i;
ferencd@0:         result += *i ++;
ferencd@0:         while(i!=s.end() && *i == *pv && *i == to_remove)
ferencd@0:         {
ferencd@0:             ++ i;
ferencd@0:         }
ferencd@0:     }
ferencd@0: 
ferencd@0:     return result;
ferencd@0: }
ferencd@0: 
ferencd@0: std::string unafrog::utils::sanitize_hostname_web(std::string hn)
ferencd@0: {
ferencd@0:     std::vector<std::pair<char, char>> replacements{
ferencd@0:         { '-', '_' },
ferencd@0:         { '.', '_' }
ferencd@0:     };
ferencd@0: 
ferencd@0:     for (auto const &r : replacements) {
ferencd@0:         hn.replace(hn.begin(), hn.end(), r.first, r.second);
ferencd@0:     }
ferencd@0: 
ferencd@0:     return hn;
ferencd@0: }
ferencd@0: 
ferencd@0: void remove_quotes(std::string &s)
ferencd@0: {
ferencd@0:     boost::trim_if(s, boost::is_any_of("\""));
ferencd@0: }