Fix streaming after Twitter API changes

bottools/methods.py

@@ -1,4 +1,4 @@
-import ast, dbtools, html, io, logging, moviepy.editor, os, PIL.Image, random, re, setuptools, string, bottools.strings, sys, telegram.ext, telegram, time, twitools, bottools.streaming, urllib.request, tweepy
+import ast, dbtools, html, io, logging, moviepy.editor, os, PIL.Image, random, re, setuptools, string, bottools.strings, sys, telegram.ext, telegram, time, twitools, bottools.streaming, bottools.thread,  urllib.request, tweepy
 
 def getTwo(message):
  try:
@@ -307,11 +307,14 @@ def explicitTweet(bot, update, args, reply = None):
    else:
     out = mobj
 
-   status = two.api.update_with_media(filename, ' '.join(args), reply, file=out)
+   status = two.api.update_with_media(filename, ' '.join(args), in_reply_to_status_id=reply, file=out)
    out.close()
 
   else:
-   status = two.tweet(' '.join(args), reply)
+   try:
+    status = two.tweet(' '.join(args), reply)
+   except Exception as e:
+    logging.exception(e)
 
   bottools.methods.tweetMessage(status, update.message.chat_id, bot)
 
@@ -328,7 +331,7 @@ def reply(bot, update, args):
   if not ("@%s" % sender.strip("@")) in [("@%s" % a.strip("@")) for a in args]:
    mentions = []
 
-   for m in re.split('[^\w@]+', otweet.text):
+   for m in re.split('[^\w@]+', otweet.full_text):
     try:
      if m[0] == "@" and m[0].strip() != "":
       mentions += [m]
@@ -464,6 +467,11 @@ def tweet(bot, update):
 def tweetMessage(status, cid, bot, force = False, callback = None, notified = None):
  db = dbtools.dbHelper()
 
+ try:
+  text = status.text
+ except:
+  text = status.full_text
+
  if not (force or callback):
   try:
    two = twitools.twoBotHelper(cid)
@@ -485,7 +493,7 @@ def tweetMessage(status, cid, bot, force = False, callback = None, notified = No
   db.commit()
 
  else:
-  i = int(callback.message.text.split()[1].strip(":"))
+  i = int(text.split()[1].strip(":"))
 
  buttons = []
  header = None
@@ -515,12 +523,15 @@ def tweetMessage(status, cid, bot, force = False, callback = None, notified = No
  if status.in_reply_to_status_id:
   buttons += [telegram.InlineKeyboardButton("View Thread", callback_data = "/thread %i" % i)]
 
- rmu = telegram.InlineKeyboardMarkup(makeMenu(buttons, header=header))
+ try:
+  rmu = telegram.InlineKeyboardMarkup(makeMenu(buttons, header=header))
+ except Exception as e:
+  logging.exception(e)
 
  if callback:
   bot.editMessageReplyMarkup(chat_id=callback.message.chat_id, message_id=callback.message.message_id, reply_markup=rmu)
  else:
-  bot.sendMessage(chat_id = cid or callback.message.chat_id, text = "Tweet %i:\n%s (@%s) at %s:\n%s" % (i, status.author.name, status.author.screen_name, status.created_at, html.unescape(status.text)), reply_markup=rmu)
+  bot.sendMessage(chat_id = cid or callback.message.chat_id, text = "Tweet %i:\n%s (@%s) at %s:\n%s" % (i, status.author.name, status.author.screen_name, status.created_at, html.unescape(text)), reply_markup=rmu)
 
 def trends(bot, update, args):
  try:
@@ -639,9 +650,10 @@ def timeline(bot, update, args = [10]):
 mentionstreams = {}
 
 def makeStream(bot, cid, ato, ase):
- two = twitools.twObject(ato=ato, ase=ase)
+# two = twitools.twObject(ato=ato, ase=ase)
- stream = tweepy.Stream(auth = two.auth, listener = bottools.streaming.BotStreamListener(bot, cid, ato))
+# stream = tweepy.Stream(auth = two.auth, listener = bottools.streaming.BotStreamListener(bot, cid, ato))
- stream.filter(track=["@%s" % two.whoami().strip("@")], async=True)
+# stream.filter(track=["%s" % two.whoami().strip("@")], async=True)
+ stream = bottools.thread.BotStreamListener(bot, cid, ato)
  return stream
  
 try:
@@ -688,10 +700,18 @@ def isadmin(message):
  return False
 
 def restart(bot, update):
+ global mentionstreams
+
  if bottools.methods.isadmin(update.message):
   update.message.reply_text(bottools.strings.restart)
-  time.sleep(0.5)
+
+  while len(mentionstreams) > 0:
+   so = mentionstreams.pop(list(mentionstreams)[0])
+   so.disconnect()
+   so.join()
+   
   os.execl(sys.executable, sys.executable, *sys.argv)
+
  else:
   bottools.methods.unknown(bot, update)
 

bottools/streaming.py

@@ -0,0 +1,15 @@
+import bottools.methods, dbtools, html, tweepy
+
+class BotStreamListener(tweepy.StreamListener):
+ def __init__(self, bot, cid, ato, *args, **kwargs):
+  tweepy.StreamListener.__init__(self, *args, **kwargs)
+  self.bot = bot
+  self.cid = cid
+  self.ato = ato
+
+ def on_status(self, status):
+  bottools.methods.tweetMessage(status, self.cid, self.bot, notified = self.ato)
+
+ def on_error(self, status):
+  if status == 420:
+   return False

bottools/thread.py

@@ -0,0 +1,41 @@
+import bottools.methods, dbtools, logging, threading, time, twitools
+
+class BotStreamThread(threading.Thread):
+ def __init__(self, bot, cid, ato, searchstring = None):
+  threading.Thread.__init__(self)
+
+  self.bot = bot
+  self.cid = cid
+  self.ato = ato
+
+  self.ase = dbtools.dbHelper().aseByAto(ato)
+  self.two = twitools.twObject(ato=ato, ase=self.ase)
+
+  self.searchstring = searchstring or "@%s" % self.two.whoami().strip("@")
+
+  self.lm = self.two.getLastMention()
+  self.stop = False
+
+ def run(self):
+  while not self.stop:
+   try:
+    for status in self.two.getMentions(self.lm):
+     try:
+      bottools.methods.tweetMessage(status, self.cid, self.bot, notified = self.ato)
+      self.lm = status.id
+     except Exception as e:
+      logging.exception(e)
+   except Exception as e:
+    logging.exception(e)
+
+   for i in range(0, 89):
+    if not self.stop:
+     time.sleep(1)
+
+ def disconnect(self):
+  self.stop = True
+
+def BotStreamListener(bot, cid, ato, searchstring = None):
+ obj = bottools.thread.BotStreamThread(bot, cid, ato, searchstring)
+ obj.start()
+ return obj

filler.py

@@ -6,18 +6,20 @@ def getTweets(db=dbtools.dbHelper(), user=twitools.twObject().whoami(), two=twit
  savepoint = db.getLatestTweet() + 1
  last = savepoint
 
- timeline = two.api.user_timeline(screen_name = user, since_id = savepoint, include_rts = True)
+ timeline = two.api.user_timeline(screen_name = user, since_id = savepoint, include_rts = True, tweet_mode = "extended")
 
  tw_counter = 0
 
  for status in timeline:
   timestamp = status.created_at.strftime('%Y-%m-%d %H:%M:%S') + " +0000"
-  text = setuptools.unescapeText(status.text)
+  text = setuptools.unescapeText(status.full_text)
 
   if filters.filler.tweetFilter(status):
    try:
     db.executeQuery("INSERT INTO tweets(tweet_id,timestamp,text) VALUES(" + str(status.id) + ",'" + timestamp + "','" + text + "')")
-   except:
+    db.commit()
+   except Exception as e:
+    print(e)
     print("Failed to insert %s into database." % str(status.id))
 
    if 'media' in status.entities:
@@ -32,8 +34,6 @@ def getTweets(db=dbtools.dbHelper(), user=twitools.twObject().whoami(), two=twit
    last = status.id
    tw_counter = tw_counter + 1
 
- db.commit()
-
  return tw_counter, last, savepoint
 
 def getMessages(db=dbtools.dbHelper(), two=twitools.twObject()):
@@ -46,6 +46,7 @@ def getMessages(db=dbtools.dbHelper(), two=twitools.twObject()):
   if filters.filler.messageFilter(m, True):
    try: 
     db.executeQuery("INSERT INTO messages VALUES(%s, '%s', %s, %s, '%s')" % (m.id, setuptools.unescapeText(m.text), m.sender_id, m.recipient_id, m.created_at))
+    db.commit()
     mcount += 1
    except:
     pass
@@ -56,8 +57,6 @@ def getMessages(db=dbtools.dbHelper(), two=twitools.twObject()):
      httptools.downloadMedia(med['media_url'], "m%i" % int(m.id), mid)
      mid += 1
 
- db.commit()
-
  return mcount, savepoint or 0, db.getLatestMessage
 
 def getFollowers(db=dbtools.dbHelper(), two=twitools.twObject(), firstrun=False):

twitools/__init__.py

@@ -14,7 +14,7 @@ class twObject:
   self.api.destroy_status(id)
 
  def search(self, query, savepoint = 0):  
-  tweets = list(tweepy.Cursor(self.api.search, q=query, since_id=savepoint, include_entities=True).items())
+  tweets = list(tweepy.Cursor(self.api.search, q=query, since_id=savepoint, result_type="recent", include_entities=True, tweet_mode="extended").items())
   tweets.reverse()
   return tweets
  
@@ -22,7 +22,7 @@ class twObject:
   return self.auth.get_username()
 
  def tweet(self, text, reply = 0):
-  return self.api.update_status(text, reply)
+  return self.api.update_status(text, in_reply_to_status_id = reply)
 
  def authenticate(self):
   try:
@@ -44,7 +44,13 @@ class twObject:
   return self.auth.access_token, self.auth.access_token_secret
 
  def getTweet(self, tid):
-  return self.api.get_status(tid)
+  return self.api.get_status(tid, tweet_mode="extended")
+
+ def getLastMention(self):
+  return self.getMentions()[-1].id
+
+ def getMentions(self, since = 0):
+  return self.search("@%s" % self.whoami().strip("@"), since)
 
 def getFollowerIDs(section = setuptools.TWITTER):
  ''' Returns 5,000 follower IDs at most '''
@@ -65,7 +71,7 @@ def getNamesByIDs(fids=getFollowerIDs(), section = setuptools.TWITTER):
    yield {"id": follower.id, "name": follower.screen_name}
 
 def getTweet(tid, section = setuptools.TWITTER):
- return twoHelper(section).api.get_status(tid)
+ return twoHelper(section).getTweet(tid)
 
 def twoHelper(section = setuptools.TWITTER):
  try: