1
require 'yaml'
2
require 'sinatra'
3
require 'rest_client'
4
require 'json'
5
require 'rack/utils'
6
#require 'oniguruma'
7
require 'oauth'
8
9
#RestClient.proxy = "http://localhost:8118"
10
#RestClient.proxy = "http://bess-proxy.mdeca.org:7033"
11
set :host, 'localhost'
12
disable :logging
13
set :views, File.dirname(__FILE__) + "/views"
14
set :public, File.dirname(__FILE__) + '/public'
15
16
configure do
17
  if File.exists? 'timeline.yml'
18
    @@data = YAML.load_file 'timeline.yml'
19
  else
20
    @@data = ""
21
  end
22
  @@config = YAML.load_file ENV['HOME'] + "/.coefficient.yml"
23
24
  consumer = OAuth::Consumer.new(@@config['credentials']['twitter']['consumer_key'],
25
                                 @@config['credentials']['twitter']['consumer_secret'],
26
                                 { :site => "https://twitter.com" })
27
28
  @@sites = {}
29
  @@sites['twitter'] = {}
30
  @@sites['twitter']['api'] = "https://twitter.com/"
31
  @@sites['twitter']['url'] = "https://twitter.com/"
32
  @@sites['twitter']['consumer'] = consumer
33
  @@sites['twitter']['access_token'] = OAuth::AccessToken.new(consumer,
34
                                                              @@config['credentials']['twitter']['access_token'],
35
                                                              @@config['credentials']['twitter']['access_secret'])
36
37
  @@sites['identica'] = {}
38
  @@sites['identica']['api'] = "https://identi.ca/api/"
39
  @@sites['identica']['url'] = "https://identi.ca/"
40
end
41
42
def make_user_link(origin, name)
43
#  origin = data['origin']
44
#  name = data['user']['screen_name']
45
  string = %Q(<a href="#{@@sites[origin]['url']}#{name}">#{name}</a>)
46
  return string
47
end
48
49
def parse_text(origin, text)
50
  if origin == "identica"
51
    return identicaify text
52
  else
53
    return twitterify text
54
  end
55
end
56
57
def escape_amp(text)
58
  return "???" if text.nil?
59
  return text.gsub(/&(?!(amp|lt|gt|#39|quot);)/, "&amp;")
60
end
61
62
def html(text)
63
  return escape_amp(text).
64
    gsub("<", "&lt;").
65
    gsub(">", "&gt;").
66
    gsub("'", "&#39;").
67
    gsub("\"", "&quot;")
68
end
69
70
def linkify(text)
71
  new = text.dup
72
  re = Regexp.new(/(https?:\/\/([\w%\-+?#\/=~;]|&(?!quot;)|[.,](?!\z|\s))+)/)
73
  new.gsub!(re, '<a href="\1">\1</a>')
74
75
  #ugh = Oniguruma::ORegexp.new('(?<!http:\/\/)(ur1.ca\/([a-zA-Z0-9_%\-+?#\/=~]|&(?!quot;)|[.,](?!\z|\s))+)')
76
  ugh = Regexp.new('(?<!http:\/\/)(ur1.ca\/([a-zA-Z0-9_%\-+?#\/=~]|&(?!quot;)|[.,](?!\z|\s))+)', Regexp::EXTENDED)
77
  new.gsub!(ugh, '<a href="http://\1">\1</a>')
78
#  new.gsub!(/(?<!http:\/\/)(ur1.ca\/([a-zA-Z0-9_%\-+?#\/=~]|&(?!quot;)|[.,](?!\z|\s))+)/, '<a href="http://\1">\1</a>')
79
  return new
80
end
81
82
def username_link(origin, text)
83
  if origin == 'twitter'
84
    return text.
85
      gsub(/@([0-9a-zA-Z_]+)/, '@<a href="http://twitter.com/\1">\1</a>')
86
  elsif origin == 'identica'
87
    return text.
88
      gsub(/@([0-9a-zA-Z_]+)/, '@<a href="http://identi.ca/\1">\1</a>')
89
  end
90
end
91
92
def hashtag_link(origin, text)
93
  if origin == 'twitter'
94
    return text.
95
      gsub(/(^|\s)#(?:(?!39;)([A-Za-z0-9_\-]+|\.(?!\z|\s)))/, '\1#<a href="http://twitter.com/search?q=\2">\2</a>')
96
  elsif origin == 'identica'
97
    return text.
98
      gsub(/(^|\s)#(?:(?!39;)([A-Za-z0-9_\-]+|\.(?!\z|\s)))/, '\1#<a href="http://identi.ca/tag/\2">\2</a>')
99
  end
100
end
101
102
def twitterify(text)
103
  text = html(text)
104
  text = linkify(text)
105
  text = username_link('twitter', text)
106
  text = hashtag_link('twitter', text)
107
108
  return text
109
end
110
111
def identicaify(text)
112
  text = html(text)
113
  text = linkify(text)
114
  text = username_link('identica', text)
115
  text = hashtag_link('identica', text)
116
117
  text.gsub!(/(?:!([A-Za-z0-9]+))/, '!<a href="http://identi.ca/group/\1">\1</a>')
118
  return text
119
end
120
121
def convert_time_to_est(time)
122
  new_time = DateTime.parse(time).new_offset(Rational(-1,6))
123
  return new_time.strftime("%A, %B %e %Y at %l:%M %P")
124
end
125
126
def request_status(origin, id)
127
  api_url = @@sites[origin]['api'].dup
128
  api_url.gsub!(/(https?:\/\/)(.+)/, '\1' <<
129
                @@config['credentials']['username'] << ":" <<
130
                @@config['credentials']['password'] << "@" <<
131
                '\2')
132
133
134
  url = api_url
135
  path = ""
136
  path << "/statuses/show/" << id << ".json"
137
  url << path
138
139
  if origin == 'twitter'
140
    puts "OAuth: getting #{path}"
141
    get = @@sites[origin]['access_token'].get(path)
142
  else
143
    puts "get #{url.sub(/\w+:\w+@/, "****:****@")}..."
144
    get = RestClient.get url, { :accept => "text/json" }
145
  end
146
147
  json = get.body
148
  parsed_data = JSON.parse json
149
150
  return parsed_data
151
end
152
153
def get_since(origin, id)
154
  page = 1
155
  received_data = Array.new
156
157
  api_url = @@sites[origin]['api'].dup
158
  api_url.gsub!(/(https?:\/\/)(.+)/, '\1' <<
159
                @@config['credentials']['username'] << ":" <<
160
                @@config['credentials']['password'] << "@" <<
161
                '\2')
162
163
164
  while (true) do
165
    url = api_url.dup
166
    path = ""
167
168
    path << "/statuses/friends_timeline.json"
169
    path << "?since_id=#{id}&include_rts=true"
170
171
    if page > 1
172
      path << "&page=#{page}"
173
    end
174
175
    url << path
176
177
    if origin == 'twitter'
178
      puts "OAuth: getting #{path}"
179
      get = @@sites[origin]['access_token'].get(path)
180
    else
181
      puts "get #{url.sub(/\w+:\w+@/, "****:****@")}..."
182
      get = RestClient.get url, { :accept => "text/json" }
183
    end
184
185
    json = get.body
186
    json.gsub!(/\003/, '')
187
    parsed_data = JSON.parse json
188
189
    if parsed_data.length == 0
190
      break
191
    end
192
193
    received_data.concat parsed_data
194
195
    # less than 20 updates means we don't need to fetch page 2
196
    #    if parsed_data.length  20
197
    #      break
198
    #    end
199
200
    page += 1
201
  end
202
203
  return received_data
204
end
205
206
def load_metadata
207
  return YAML.load_file("metadata.yml")
208
end
209
210
def save_metadata(metadata)
211
  File.open("metadata.yml", "w") do |file|
212
    file.puts metadata.to_yaml
213
  end
214
end
215
216
def save_timeline(data)
217
  File.open("timeline.yml", "w") do |file|
218
    file.puts data.to_yaml
219
  end
220
end
221
222
def update_metadata(metadata, data)
223
  metadata.keys.each do |site|
224
    first = data.find {|obj| obj['origin'] == site }
225
    if first.nil?
226
      metadata[site]['new_since'] = metadata[site]['last_id']
227
      next
228
    end
229
    metadata[site]['last_update'] = convert_time_to_est(first['created_at'])
230
    metadata[site]['last_request'] = convert_time_to_est(Time.now.to_s)
231
    metadata[site]['new_since'] = metadata[site]['last_id']
232
    metadata[site]['last_id'] = first['id']
233
  end
234
  return metadata
235
end
236
237
get '/' do
238
  @metadata = load_metadata()
239
  @data = @@data
240
  erb :"timeline.html"
241
#  "whatever"
242
end
243
244
post '/update' do
245
  metadata = load_metadata()
246
  new_entries = []
247
248
  req_status = {}
249
250
  metadata.keys.each do |site|
251
    req_status[site] = {}
252
    begin
253
      new_data = get_since(site, metadata[site]['last_id'])
254
      # XXX Does this catch 502's?
255
      # XXX on 28 April 2009 we got a Errno::ECONNRESET - Connection reset
256
      # by peer - SSL_connect
257
      # XXX: OpenSSL::SSL::SSLError - SSL_connect SYSCALL returned=5
258
      # errno=0 state=SSLv2/v3 read server hello A:
259
260
      # sometimes, twitter generates
261
      # x = JSON.parse('{"request":NULL,"error":"Twitter is down for maintenance. It will return in about an hour."}')
262
      # which throws JSON::ParserError because NULL is different than null.
263
      # RestClient::RequestTimeout: RestClient::RequestTimeout
264
    rescue RestClient::Exception => e
265
      req_status[site]['msg'] = "#{site}: error: #{e.http_code}"
266
      req_status[site]['error'] = true
267
    rescue SocketError => e
268
      req_status[site]['msg'] = "#{site}: error: #{e}"
269
      req_status[site]['error'] = true
270
    rescue Errno::ETIMEDOUT => e
271
      req_status[site]['msg'] = "#{site}: timed out: #{e}"
272
      req_status[site]['error'] = true
273
    rescue Errno::ECONNREFUSED => e
274
      req_status[site]['msg'] = "#{site}: connection refused"
275
      req_status[site]['error'] = true
276
    rescue Net::HTTPFatalError => e
277
      req_status[site]['msg'] = "#{site}: fatal error (#{e})"
278
      req_status[site]['error'] = true
279
    end
280
281
    # XXX What happens if page 1 comes back, but page 2 errors out?
282
283
    if not new_data.nil? and new_data.length > 0
284
      new_data.each do |entry|
285
        entry['origin'] = site
286
      end
287
      req_status[site]['msg'] = "#{site}: success"
288
      req_status[site]['error'] = false
289
      new_entries.concat new_data
290
    elsif not new_data.nil? and new_data.length == 0
291
      req_status[site]['msg'] = "#{site}: nothing new"
292
      req_status[site]['error'] = false
293
    end
294
  end
295
296
#  result = RubyProf.stop
297
#  printer = RubyProf::FlatPrinter.new(result)
298
#  printer.print(STDOUT, 0)
299
300
  if new_entries.length == 0
301
    # for whatever reason, there was no new data
302
    @req_status = req_status
303
    p @req_status
304
    return erb :"update.html"
305
  end
306
307
  new_data = new_entries.sort_by { |entry|
308
    DateTime.parse(entry['created_at'])
309
  }.reverse
310
311
  metadata = update_metadata(metadata, new_data)
312
  new_data.concat(@@data)
313
  @@data = new_data
314
  save_timeline(@@data)
315
  save_metadata(metadata)
316
  @req_status = req_status
317
  return erb :"update.html"
318
end
319
320
get '/trim' do
321
  new_data = @@data.slice(0..74)
322
  @@data = new_data
323
  save_timeline(@@data)
324
  redirect '/'
325
end
326
327
get '/status/:origin/:id' do
328
  puts "something got /status/..."
329
  # XXX This does no exception handling at all!
330
  @status = request_status params[:origin], params[:id]
331
  if @status.has_key? 'error'
332
    return "Error!"
333
  else
334
    erb :"status.html"
335
  end
336
end
337
338
post '/post' do
339
  text = params[:post_textarea]
340
  in_reply_to = params[:in_reply_to]
341
  destination = params[:destination]
342
  url = ""
343
344
  if text.length > 140
345
    halt 403, "text too long: #{text.length} characters."
346
  end
347
348
  if in_reply_to.empty?
349
    send_in_reply_to = false
350
  else
351
    send_in_reply_to = true
352
  end
353
354
  url = @@sites[destination]['api'].dup
355
  url.gsub!(/(https?:\/\/)(.+)/, '\1' <<
356
            @@config['credentials']['username'] << ":" <<
357
            @@config['credentials']['password'] << "@" <<
358
            '\2')
359
360
  url << "/statuses/update.json"
361
362
  payload = {}
363
  payload[:status] = text
364
365
  if send_in_reply_to
366
    payload[:in_reply_to_status_id] = in_reply_to
367
  end
368
369
  begin
370
    res = RestClient.post url, payload
371
    result = res.body
372
  rescue Exception => e
373
    halt 403, "Exception from RestClient: #{e}"
374
  end
375
376
  begin
377
    JSON.parse(result)
378
  rescue Exception => e
379
    halt 403, "JSON failed to parse, may have submitted okay (#{e})"
380
  end
381
382
  return "success"
383
end