0xDEADBEEF

hypertweeter

Humane twit­ter client, ver­sion 3.0

wget -nc https://repo1.maven.org/maven2/org/twitter4j/twitter4j-core/4.0.7/twitter4j-core-4.0.7.jar
wget https://deadbeef.k47.cz/t/tw.scala
scala -cp twitter4j-core-4.0.7.jar tw.scala

Java 11 and Scala 2.13 requi­red.

tw.scala

/** license: MPL 2.0 */

import java.io._
import java.net._
import java.nio.charset._
import java.nio.file._
import java.security.MessageDigest
import java.time._
import java.time.format._
import java.util.{ Date, Scanner }
import com.sun.net.httpserver._
import scala.collection.mutable
import scala.jdk.CollectionConverters._
import twitter4j._

// application key and secret "borrowed" from TTYtter (be nice to them)
val key    = "XtbRXaQpPdfssFwdUmeYw"
val secret = "csmjfTQPE8ZZ5wWuzgPJPOBR9dyvOBEtHT5cJeVVmAA"

// how often the timeline should be updated
// twitter API allow at most 15 updates in a 15 minute period
val updatePeriodInSeconds = 8*60

// Add your own filtering rules to this method.
// It's called on every tweet and if it returns true, tweet is grayed out.
def grayoutTweet(t: Status) = {
  (t.getUser.getScreenName, rawTweetText(t).toLowerCase) match {
    case _ => false
  }
}

def hideTweet(t: Status) = {
  false
}

val rssFeeds = Seq(
)
val rssUpdatePeriodInSeconds = 2*60*60

val html5pages = Seq(
)

// turn off twitter, useful if you want to use only RSS
val useTwitter = true


val offline = args.length >= 1 && args(0) == "offline"

// everything must be utf-8
val charset = Charset.forName("utf-8")


val accessTokenFile = new File("tw.keys")

if (!accessTokenFile.exists) {
  val twitter = TwitterFactory.getSingleton()
  twitter.setOAuthConsumer(key, secret)
  val requestToken = twitter.getOAuthRequestToken()
  var accessToken: auth.AccessToken = null
  val br = new BufferedReader(new InputStreamReader(System.in))
  while (null == accessToken) {
    println("Open the following URL and grant access to your account:")
    println(requestToken.getAuthorizationURL())
    print("Enter the PIN if aviailable or just hit enter:")
    val pin = br.readLine()
    try {
      if (pin.length > 0) {
        accessToken = twitter.getOAuthAccessToken(requestToken, pin)
      } else {
        accessToken = twitter.getOAuthAccessToken()
      }
    } catch {
      case e: TwitterException =>
        if (e.getStatusCode == 401) {
          println("Unable to get the access token.")
        } else {
          e.printStackTrace()
        }
        sys.exit()
    }
  }

  val fw = new FileWriter(accessTokenFile, charset)
  fw.write(accessToken.getToken()+"\n")
  fw.write(accessToken.getTokenSecret())
  fw.close()
}

val Seq(accessToken, accessTokenSecret) = io.Source.fromFile(accessTokenFile).getLines.toSeq


val twitter = {
  val cb = new conf.ConfigurationBuilder()
    .setDebugEnabled(true)
    .setOAuthConsumerKey(key)
    .setOAuthConsumerSecret(secret)
    .setOAuthAccessToken(accessToken)
    .setOAuthAccessTokenSecret(accessTokenSecret)
  new TwitterFactory(cb.build()).getInstance
}



/** Timeline class takes care of updating the timeline from twitter API,
 *  loading and saving the timeline to disc, downloading images to disc and
 *  managing watermark. */
class Timeline {
  @volatile private[this] var _timeline: Vector[Status] = null
  @volatile private[this] var _watermark: Long = 0

  // changes to the map must synchronize on it
  private val feeds: mutable.Map[String, Seq[FeedItem]] =
    rssFeeds.map { url => (url, Seq[FeedItem]()) }.to(mutable.Map)

  def timeline: Vector[Status] = _timeline
  def watermark: Long = _watermark
  def maxId = _timeline.map(_.getId).max

  def allFeedItems: Seq[FeedItem] = feeds.synchronized {
    feeds.map(_._2).flatten.toVector
  }

  private val imageDirectory = new File("tw")
  private val watermarkFile = new File("tw.watermark")
  private val timelineFile  = new File("tw.timeline")

  def update(initialization: Boolean = false): Unit = this.synchronized {
    println("updating timeline")

    if (initialization) {
      _watermark = if (!watermarkFile.exists) 0 else {
        io.Source.fromFile(watermarkFile, charset.name).mkString.trim.toLong
      }
    }

    // load old snapshot
    val oldTweets = if (!timelineFile.exists) Vector() else {
      val oldTweets = deserialize(timelineFile)
      if (oldTweets == null) Vector() else oldTweets.asInstanceOf[scala.collection.Seq[Status]].toVector
    }

    // fetch new tweets, API limit 15 calls in 15 minutes
    val since = if (oldTweets.isEmpty) 1L else oldTweets.map(_.getId).max
    val newTweets = if (offline) Vector() else twitter.getHomeTimeline(new Paging(1, 200).sinceId(since)).asScala.toVector

    if (newTweets.nonEmpty) {
      println(""+newTweets.size+" new tweets")
    }

    // merge them together
    val timeline = newTweets ++ oldTweets

    // save snapshot, keep only some number of lastest tweets
    serialize(timelineFile, timeline.take(3000).toVector)

    imageDirectory.mkdirs()

    for (t <- timeline.take(400)) {
      downloadAllImages(t)
    }

    _timeline = timeline
  }

  def setWatermark(watermark: Long) = this.synchronized {
    // update in memory watermark
    _watermark = watermark

    // update persistent watermark
    val fw = new FileWriter(watermarkFile, charset)
    fw.write(watermark.toString)
    fw.close()
  }

  private def downloadAllImages(t: Status): Unit = {
    for (e <- t.getMediaEntities) {
      e.getType match {
        case "photo" => downloadImage(e.getMediaURLHttps, "tw/"+md5(e.getMediaURLHttps))
        case _ =>
      }
    }
    if (t.getRetweetedStatus != null) downloadAllImages(t.getRetweetedStatus)
    if (t.getQuotedStatus != null) downloadAllImages(t.getQuotedStatus)
  }

  private def downloadImage(src: String, dest: String): Unit = {
    if (new File(dest).exists()) return

    println(s"copy $src -> $dest")
    val in = new URL(src).openStream()
    Files.copy(in, Paths.get(dest), StandardCopyOption.REPLACE_EXISTING)
  }

  def updateFeeds() = {
    rssFeeds.foreach   { url => updateFeed(url, Feeds.parseFeedFromURL _)  }
    html5pages.foreach { url => updateFeed(url, Feeds.parseHtml5 _) }
  }


  def updateFeed(feedUrl: String, getItems: String => Seq[FeedItem]) = {
    val feedFile = new File(imageDirectory, "feed-"+md5(feedUrl))

    val oldItems = if (!feedFile.exists) Vector() else {
      try {
        deserializeRSV(feedFile) { fields =>
          val Seq(feedTitle, feedUrl, title, url, date, text, timelineTime) = fields
          FeedItem(Feed(feedTitle, feedUrl), title, url, LocalDateTime.parse(date), text, timelineTime = timelineTime.toLong)
        }
      } catch {
        case e: Exception =>
          println("deserialization error: "+e)
          Vector()
      }
    }

    val oldUrls = oldItems.map(_.url).toSet

    val tt = {
      val tl = _timeline
      Math.max(_watermark+1, if (tl.isEmpty) 0L else tl.maxBy(_.getId).getId)
    }
    val newItems = getItems(feedUrl)
      .filter(i => !oldUrls.contains(i.url))
      .map { _.copy(timelineTime = tt) } // mark them after the current watermark, so they stay visible

    val allItems = oldItems ++ newItems

    serializeRSV[FeedItem](feedFile, allItems) { item =>
      Seq(item.feed.title, item.feed.url, item.title, item.url, item.date.toString, item.text, item.timelineTime.toString)
    }

    feeds.synchronized {
      feeds(feedUrl) = allItems
    }
  }


  private def deserialize(f: File): Object = {
    val fin = new FileInputStream(f)
    val ois = new ObjectInputStream(fin)
    val data = try {
      ois.readObject()
    } catch {
      case e: Exception => null
    }
    ois.close()
    fin.close()
    data
  }

  private def serialize(f: File, data: Object): Unit = {
    val fos = new FileOutputStream(f)
    val out = new ObjectOutputStream(fos)
    out.writeObject(data)
    out.close()
    fos.close()
  }

  private val recordSeparator = "\u001E" // ASCII record separator

  private def serializeRSV[T](file: File, data: Seq[T])(f: T => Seq[String]) = {
    val fw = new FileWriter(file, charset)
    for (x <- data) {
      for (field <- f(x)) {
        fw.write(field.replace(recordSeparator, "").replace("\\", "\\\\").replace("\n", "\\n"))
        fw.write(recordSeparator)
      }
      fw.write("\n")
    }
    fw.close()
  }

  private def deserializeRSV[T](file: File)(f: Seq[String] => T) = {
    io.Source.fromFile(file).getLines.map { line =>
      f(line.split(recordSeparator).map { field =>
        field.replace("\\\\", "\\").replace("\\n", "\n") // TODO is this correct?
      }.toSeq)
    }.toVector
  }
}


def md5(txt: String) = {
  val hash = MessageDigest.getInstance("MD5").digest(txt.getBytes("utf-8"))
  val hex = "0123456789abcdef".toCharArray
  val chars = new Array[Char](hash.length * 2)
  for (j <- 0 until hash.length) {
      val v = hash(j) & 0xFF
      chars(j * 2) = hex(v >>> 4)
      chars(j * 2 + 1) = hex(v & 0x0F)
  }
  new String(chars)
}

/** Reply train represents few tweets smushed together (typically train of
 *  replies or dreaded twitter thread. */
sealed trait ReplyTrain {
  def maxId: Long
}
case class TweerReplyTrain(tweets: Seq[Status]) extends ReplyTrain {
  val maxId: Long = tweets.map(_.getId).max
}
case class FeedItemTrain(item: FeedItem) extends ReplyTrain {
  require(item.timelineTime != -1, item)
  def maxId = item.timelineTime
}

def makeUnifiedTimeline(timeline: Timeline): Seq[ReplyTrain] =
  ((groupReplies(timeline.timeline.take(250)) ++ timeline.allFeedItems.map(FeedItemTrain)) : Seq[ReplyTrain])
    .sortBy(~_.maxId)
    .filter {
      case TweerReplyTrain(tweets) => !hideTweet(tweets.head)
      case FeedItemTrain(item) => true
    }


def groupReplies(tweets: Seq[Status]): Seq[TweerReplyTrain] = {
  val trainIds  = mutable.ArrayBuffer[Long]()
  val trainMap  = mutable.Map[Long, Seq[Status]]()
  val redirects = mutable.Map[Long, Long]()

  for (t <- tweets.reverse) {
    val id = t.getId
    val replyTo = t.getInReplyToStatusId

    // reply to start of a train
    if (trainMap.contains(replyTo)) {
      trainMap(replyTo) = trainMap(replyTo) :+ t
      redirects(id) = replyTo

    } else if (redirects.contains(replyTo)) {
      trainMap(redirects(replyTo)) = trainMap(redirects(replyTo)) :+ t
      redirects(id) = redirects(replyTo)

    // a reply to nothing or a reply to tweet outside of our dataset
    // start a brand new train
    } else {
      trainMap(id) = Seq(t)
      trainIds += id
    }
  }

  trainIds.map(id => TweerReplyTrain(trainMap(id))).reverse.toVector
}


def escape(str: String)   = str.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;").replace("\"", "&quot;").replace("'",  "&#x27;")
def unescape(str: String) = str.replace("&amp;", "&").replace("&lt;", "<").replace("&gt;", ">").replace("&quot;", "\"").replace( "&#x27;", "'")

def mkTweetBody0(t: Status, compactSelfReply: Boolean = false): String = {
  var txt =
    if (t.getRetweetedStatus != null) {
      val u = t.getRetweetedStatus.getUser.getScreenName
      val rt = mkTweetBody0(t.getRetweetedStatus)
      "RT <s>@"+u+"</s>: "+rt
    } else {
      escape(t.getText)
    }

  if (t.getInReplyToScreenName != null && !compactSelfReply) {
    val name = t.getInReplyToScreenName
    val id   = t.getInReplyToStatusId
    txt = "<a href="+statusUrl(name, id)+">⮆"+name+"</a> "+txt
  }

  if (t.getQuotedStatus != null) {
    val p = t.getQuotedStatusPermalink
    txt = txt.replace(p.getURL, " ▒ @"+t.getQuotedStatus.getUser.getScreenName+": "+mkTweetBody0(t.getQuotedStatus))
  }

  for (e <- t.getURLEntities) {
    val r = """^https?://(?:www\.)?([^/]+.{0,12})(.*)$""".r
    val domain = e.getExpandedURL match {
      case r(base, "") => base
      case r(base, _) => base+"…"
      case x => x
    }
    val url = e.getExpandedURL
      .replaceAll("#utm.*", "")
      .replaceAll("[?&]fbclid=[^?&]+", "")
      .replaceAll("[?&]utm_[^?&]+", "")

    txt = txt.replace(e.getURL, "<a href='"+url+"'>"+domain+"</a>")
  }

  for (e <- t.getMediaEntities) {
    val repl = e.getType match {
      case "photo"        => "<a class=inv href='tw/"+md5(e.getMediaURLHttps)+"'>IMAGE</a>"
      case "video"        => "<a class=inv href='"+statusUrl(t)+"'>VIDEO</a>"
      case "animated_gif" => "<a class=inv href='"+e.getMediaURLHttps+"'>GIF</a>"
    }
    txt = txt.replace(e.getURL, repl)
  }

  txt.replaceAll("(?U)(?<=\\s)#\\w+", "<s>$0</s>")
     .replaceAll("(?U)(?<=\\s)@\\w+", "<s>$0</s>")
}

def mkTweetBody(t: Status, compactSelfReply: Boolean = false): String =
  mkTweetBody0(t, compactSelfReply).replaceAll("\n+", " ⧸⧸ ")

def popupImages(tweets: Seq[Status]) = {
  val images = tweets
    .flatMap(flattenSubtweets)
    .flatMap(_.getMediaEntities)
    .collect { case e if e.getType == "photo" => e.getMediaURLHttps }
    .distinct

  "<div class=popup>"+
  images.map { img => "<img loading=lazy src=\"tw/"+md5(img)+"\">" }.mkString+
  "</div>"
}


def flattenSubtweets(t: Status): Seq[Status] =
  if (t == null) Seq() else t +: flattenSubtweets(t.getRetweetedStatus) ++: flattenSubtweets(t.getQuotedStatus)


def rawTweetText(t: Status): String = {
  var txt =
    if (t.getRetweetedStatus != null) {
      rawTweetText(t.getRetweetedStatus)
    } else {
      t.getText
    }

  if (t.getQuotedStatus != null) {
    txt = txt + " " + rawTweetText(t.getQuotedStatus)
  }

  txt
}

def linksInTweet(t: Status) =
  flattenSubtweets(t).flatMap(_.getURLEntities).map(_.getExpandedURL)



def statusUrl(screenName: String, id: Long): String =
  "https://twitter.com/"+screenName+"/status/"+id

def statusUrl(t: Status): String =
  if (t.getRetweetedStatus != null) statusUrl(t.getRetweetedStatus) else statusUrl(t.getUser.getScreenName, t.getId)

def linkToStatus(t: Status) =
  "<a href=\""+statusUrl(t)+"\">⮹</a>"






case class Feed(title: String, url: String) extends Serializable
case class FeedItem(
  feed: Feed, title: String, url: String, date: LocalDateTime, text: String,
  commentsUrl: String = "", commentsRssUrl: String = "", // TODO serialize/deserialize
  timelineTime: Long = -1
) extends Serializable


object Feeds {
  import javax.xml.parsers._
  import javax.xml.xpath._
  import org.w3c.dom._

  val builder = {
    val factory = DocumentBuilderFactory.newInstance()
    factory.setCoalescing(true)
    factory.setNamespaceAware(true)
    factory.newDocumentBuilder()
  }

  val xpath = XPathFactory.newInstance.newXPath

  def matchNodes(n: Node, path: String) = {
    val nodes = xpath.evaluate(path, n, XPathConstants.NODESET).asInstanceOf[NodeList]
    (0 until nodes.getLength).map(nodes.item)
  }

  def crunchText(txt: String) =
    txt.replace("\u00AD", "")
      .replaceAll("</?\\w+[^>]*?>", "")
      .replaceAll("\\s+", " ")
      .take(150)
      .replaceAll("(?U) ?\\w+\\Z", "…")

  def parseFeedFromURL(url: String): Seq[FeedItem] = {
    val doc = builder.parse(url)
    val root = doc.getDocumentElement

    root.getNodeName match {
      case "rss" =>
        val channel = root.getChildNodes.item(0)
        require(channel.getNodeName == "channel")

        val title = xpath.evaluate("title", channel)
        val link  = xpath.evaluate("link", channel)
        val feed = Feed(title, link)

        for (item <- matchNodes(channel, "item")) yield {
          val title = xpath.evaluate("title", item)
          val guid  = xpath.evaluate("guid", item)
          val link  = xpath.evaluate("link", item)
          val _date = xpath.evaluate("pubDate", item)
          val desc  = xpath.evaluate("description", item)

          val url = if (link.nonEmpty) link else guid
          val date = LocalDateTime.parse(_date, DateTimeFormatter.RFC_1123_DATE_TIME)
          val txt = crunchText(desc)

          FeedItem(feed, title, if (link.nonEmpty) link else guid, date, txt)
        }

      case "atom" =>
        val title = xpath.evaluate("title", root)
        val link  = xpath.evaluate("link", root)
        val feed = Feed(title, link)

        for (item <- matchNodes(root, "entry")) yield {
          val title = xpath.evaluate("title", item)
          val guid  = xpath.evaluate("id", item)
          val link  = xpath.evaluate("link[@type='text/html']/@href | link/@href", item)
          val _date = xpath.evaluate("updated", item)
          val desc  = xpath.evaluate("content/text()", item)

          val url = if (link.nonEmpty) link else guid
          val date = LocalDateTime.parse(_date, DateTimeFormatter.ISO_OFFSET_DATE_TIME)
          val txt = crunchText(desc)

          FeedItem(feed, title, if (link.nonEmpty) link else guid, date, txt)
        }
    }
  }

  private val titleRegex = """<title>(.*?)</title>""".r
  private val articleRegex = """(?s)<article(.*?)</article>""".r
  private val aHref = """<a[^>]*href=([^\\ >]+)[^>]*>"""
  private val headingRegex = s"""(?x) (?: $aHref \\s*)? <h([123456])> (?: \\s* $aHref )? (.*?)</h\\d>""".r

  def parseHtml5(pageUrl: String): Seq[FeedItem] = {
    val htmlPage = io.Source.fromURL(pageUrl, charset.name).mkString
    val title = titleRegex.findFirstMatchIn(htmlPage).map(_.group(1)).getOrElse(pageUrl)
    val feed = Feed(title, pageUrl)

    (for (m <- articleRegex.findAllMatchIn(htmlPage).toSeq; txt = m.group(1)) yield {

      val hs = (for (m <- headingRegex.findAllMatchIn(txt)) yield {
        val preHref  = m.group(1)
        val level    = m.group(2).toInt
        val postHref = m.group(3)
        val title    = m.group(4)
        val href     = if (preHref != null) preHref else postHref
        if (href == null) null else (level, (new URI(pageUrl).resolve(href).toString, stripTags(title)))
      }).filter(_ != null)

      if (hs.isEmpty) None else {
        val (href, title) = hs.minBy(_._1)._2
        Some(FeedItem(feed, title, href, LocalDateTime.now(), ""))
      }

    }).flatten
  }
}



def stripTags(txt: String) = txt.replaceAll("<[^>]+>", "")

val symbols = Array("🞅","🞊","🞋","🞎","🞓","🞕","🞖","🞙","🞛","🞜","🞠","🞤","🞧","🞮","🞰","🞴","🞻","🞿","🟅","🟋","🟔","■", "▤", "▩", "▰", "▲", "◌", "◐", "◒")
def userSymbol(t: Status) = {
  val h = t.getUser.getScreenName.hashCode
  val s = symbols(Math.abs(h % symbols.length))
  val c = h.toHexString.substring(0, 6)
  "<span style=color:#"+c+">"+s+s+s+"</span>"
}

def renderTimeline(timeline: Timeline, reverse: Boolean = false): String = {
  val watermark = timeline.watermark

  val tl1 = makeUnifiedTimeline(timeline)
  val tl2 = if (reverse) tl1.reverse else tl1

  tl2.map {
    case FeedItemTrain(item) =>
      val res =
        "<div class=tw>"+
        s"""⌘⌘⌘ <a href="${item.feed.url}"><b>${item.feed.title}</b></a> - <a href="${item.url}">${item.title}</a>: ${item.text}"""+
        "</div>"

        val gray = item.timelineTime <= watermark
        if (gray) s"<s>$res</s>" else res

    case TweerReplyTrain(tweets) =>

      var prevTweet: Status = null

      "<div class=tw>"+
      tweets.map { t =>

        // shortened reply version of a tweet
        val res = if (prevTweet != null && prevTweet.getUser.getScreenName == t.getUser.getScreenName) {
          s" ${linkToStatus(t)} ${mkTweetBody(t, true)}"

        // full tweet form
        } else {
          val name = " <b>"+t.getUser.getScreenName+"</b>"
          s"${userSymbol(t)}$name ${linkToStatus(t)} ${mkTweetBody(t)}"
        }

        prevTweet = t

        val gray = t.getId <= watermark || grayoutTweet(t)
        if (gray) s"<s>$res</s>" else res


      }.mkString("\n")+
      popupImages(tweets)+
      "</div>"

  }.mkString("\n")
}



class HomeHandler extends HttpHandler {
  def makeResponse() = {
    val r = renderTimeline(timeline)
    val maxId = timeline.maxId

s"""<!DOCTYPE html>
<html>
<meta charset=utf-8>
<title>hypertweeter</title>
<meta name="referrer" content="no-referrer">
<link rel=alternate type=application/rss+xml href=rss title="RSS export">
<style>
s { text-decoration: none; color: gray;}
body { font-family: sans; font-size: 0.85em; line-height: 1.45; background-color: #171616; color: white; }
main { max-width: 50em; margin-left: 8em; }
a { color: #00b5b5; text-decoration: none; }
a.inv { text-decoration: none; border: 1px solid #00b5b5; padding: 1px 2px 0; }
del, del a, del s { text-decoration: none; color: #333; }
.tw { padding: 0.35em; }
.tw:hover { background-color: rgba(150, 150, 150, 0.12); }
.n { color: #555; }
.popup { position: fixed; top: 0; right: 0; display: none; width: 35em; height: 100%; }
.popup img { max-width:30em; max-height:30em; float: right; }
.tw:hover .popup { display: block; }
</style>
<script>
  let dark = localStorage.getItem("color") !== "light";

  if (!dark) {
    let ss = document.styleSheets[0];
    ss.insertRule("body {background-color: white; color: black; }", ss.rules.length);
  }

  function toggle() {
    dark = !dark;
    localStorage.setItem("color", dark ? "dark" : "light");

    let s = document.body.style;
    s.backgroundColor = dark ? "#171616" : "white";
    s.color           = dark ? "white"   : "black";
  }
</script>
<main>
<div>
  <div onclick="toggle()" style="position:absolute;top:0;right:0;padding-right:0.5em">
    dark / light mode
  </div>
  <form method="post" action="/post">
    <textarea style="width:100%;background-color:#444;border:1px solid black;" id="tweet" name="tweet"></textarea>
    <input style="float:right;background-color:#444;border:1px solid black;" type="submit" value="tweet" />
  </form>
  <div style="clear:both; text-align:center; padding:0.5em;"><a href=mark?$maxId>mark everything as read</a></div>
  <hr style="clear:both; visibility:hidden; margin:0;" />
</div>
$r
</main>
"""
  }


  def handle(ex: HttpExchange): Unit = {
    val bytes = try {
      makeResponse().getBytes(charset)
    } catch {
      case e: Exception =>
        e.toString.getBytes(charset)
    }

    ex.sendResponseHeaders(200, bytes.length)
    val os = ex.getResponseBody()
    os.write(bytes)
    os.close()
  }
}



class RssHandler extends HttpHandler {
  def getItems = {
    makeUnifiedTimeline(timeline).map {
      case FeedItemTrain(item) => item
      case TweerReplyTrain(tweets) =>
        val t = tweets.head
        val feed = Feed("twitter", "https://twitter.com")
        val title = "@"+t.getUser.getScreenName
        val url = statusUrl(t)
        val date = LocalDateTime.ofInstant(t.getCreatedAt.toInstant, ZoneId.systemDefault)
        val text =
          tweets.map { t =>
            s"<b>${t.getUser.getScreenName}</b> ${linkToStatus(t)} ${mkTweetBody(t)}<br>"
          }.mkString("\n")
          //+popupImages(tweets)

        FeedItem(feed, title, url, date, text)
    }
  }

  def writeRSS(os: OutputStream, items: Seq[FeedItem]) = {
    import javax.xml.stream._

    val w = XMLOutputFactory.newInstance.createXMLStreamWriter(os)

    def tag(w: XMLStreamWriter, tag: String, content: String, attrs: Map[String, String] = Map()) = {
      w.writeStartElement(tag)
      for ((k, v) <- attrs) {
        w.writeAttribute(k, v)
      }
      w.writeCData(content)
      w.writeEndElement()
    }

    def tagTree(w: XMLStreamWriter, tag: String, attrs: Map[String, String] = Map())(f: XMLStreamWriter => Unit) = {
      w.writeStartElement(tag)
      for ((k, v) <- attrs) {
        w.writeAttribute(k, v)
      }
      f(w)
      w.writeEndElement()
    }

    val format = DateTimeFormatter.RFC_1123_DATE_TIME
    def rssdate(date: LocalDateTime) = if (date == null) "" else format.format(date.atZone(java.time.ZoneId.systemDefault))

    w.writeStartDocument()

    tagTree(w, "rss", Map("version" -> "2.0")) { w =>
      tagTree(w, "channel") { w =>
        tag(w, "title", "twitter")
        tag(w, "description", "")
        tag(w, "link", "")

        for (item <- items) {
          tagTree(w, "item") { w =>
            tag(w, "title", item.title)
            tag(w, "guid", item.url, Map("isPermaLink" -> "true"))
            tag(w, "pubDate", rssdate(item.date))
            tag(w, "description", item.text)
          }
        }
      }
    }

    w.writeEndDocument()
    w.close()
  }


  def handle(ex: HttpExchange): Unit = try {
    ex.getResponseHeaders.add("Content-Type", "application/rss+xml");
    ex.sendResponseHeaders(200, 0)
    val os = ex.getResponseBody()
    writeRSS(os, getItems)
    os.close()
  } catch {
    case e: Exception => println(e)
  }
}


val fileHandler: HttpHandler = (ex: HttpExchange) =>
  try {
    val path = ex.getRequestURI.getPath

    ex.sendResponseHeaders(200, 0)
    val os = ex.getResponseBody()

    try {
      Files.copy(new File(".", path).toPath, os)
    } finally {
      os.close()
    }

  } catch {
    case e: Exception => println(e)
  }



val watermarkHandler: HttpHandler = (ex: HttpExchange) => {
  try {
    val wm = ex.getRequestURI.getQuery.toLong
    timeline.setWatermark(wm)
  } catch  {
    case e: Exception => println(e)
  }
  ex.getResponseHeaders.add("Location", "/home")
  ex.sendResponseHeaders(303, -1)
}


val postHandler: HttpHandler = (ex: HttpExchange) => {
  try {
    val is = ex.getRequestBody()
    val scanner = new Scanner(is).useDelimiter("\\A")
    val result = if (scanner.hasNext) scanner.next() else ""

    val params = result.split("&")
      .map { x => val Array(k, v) = x.split("=", 2); (k, v) }
      .toMap

    val tweet = URLDecoder.decode(params("tweet"), charset)

    if (tweet.nonEmpty) {
      twitter.updateStatus(new StatusUpdate(tweet))
    }

  } catch  {
    case e: Exception => println(e)
  }
  ex.getResponseHeaders.add("Location", "/home")
  ex.sendResponseHeaders(303, -1)
}




// initialize timeline
val timeline = new Timeline
if (useTwitter) {
  timeline.update(true)
}


// this thread periodically updates the timeline
if (!offline) {
  if (useTwitter) {
    new Thread(() => {
      while (true) {
        Thread.sleep(updatePeriodInSeconds*1000)
        try {
          timeline.update()
        } catch {
          case e: Exception => println(e)
        }
      }
    }).start()
  }

  if (rssFeeds.nonEmpty || html5pages.nonEmpty) {
    new Thread(() => {
      while (true) {
        try {
          timeline.updateFeeds()
        } catch {
          case e: Exception => println(e)
        }
        Thread.sleep(rssUpdatePeriodInSeconds*1000)
      }
    }).start()
  }
}


// start server
val server = HttpServer.create(new InetSocketAddress(8889), 0)
server.createContext("/home", new HomeHandler())
server.createContext("/rss", new RssHandler)
server.createContext("/tw", fileHandler)
server.createContext("/mark", watermarkHandler)
server.createContext("/post", postHandler)
server.setExecutor(null)
server.start()


// everything is running, happy browsing
println("client running on localhost:8889")



val console = System.console()
if (console != null) {
  while (true) {
    val str = console.readLine("")
    println(unescape(stripTags(renderTimeline(timeline, reverse = true).replaceAll("<br\\?>", "\n"))))
    timeline.setWatermark(timeline.maxId)
    println("\n*\n")
  }
}