#!/usr/bin/env python
#coding: utf-8
#
# scdd.py daemon process
#
# author: observer
# email: jingchaohu@gmail.com
# blog: http://obmem.com
# last edit @ 2009.12.23
import os,sys,time
import re
#from daemon import Daemon
import sqlite3,MySQLdb
import fetchvc_noth as fetchvc
import fcmt_noth as fcmt
import douban
import download
import urllib2
from Queue import Queue
from threading import Thread,stack_size,Lock
from random import randint
from feed import feed
import memcache


mc = memcache.Client(['127.0.0.1:11211'], debug=0)
stack_size(32768*32)
dico = {}

class FetchManger:
    def __init__(self,path):
        self.path = path
        self.q = Queue()
        self.running = False

    def ensure_dir(self,f):
        d = os.path.dirname(f)
        if not os.path.exists(d):
            os.makedirs(d)
    
    def thread_fetch(self):
        conn = sqlite3.connect(self.path+'/verycd.sqlite3.db')
        conn2 = MySQLdb.connect(user='root',passwd='guess8',db='simplecd')
        dbl = sqlite3.connect(self.path+'/lock.sqlite3.db')
        statdb = conn2
        conn.text_factory = str
        conn2.text_factory = str
        dbl.text_factory = str
        #download.httpfetch('http://www.verycd.com',needlogin=False)
        while True:
            topic,updtime = self.q.get()
            try:
                #print 'fetching topic',topic,'...'
                rtn = fetchvc.fetch(topic,conn=conn,dbl=dbl,statdb=statdb,needlogin=False,updtime=updtime)
                if not rtn:
                    self.q.task_done()
                    print '__cannot access the resource'
                    continue
                print 'fetching cmt',topic,'...'
                fcmt.fetchcmt(topic,conn2,needlogin=False)
                print 'fetching douban',topic,'...'
                douban.douban(topic,conn=conn)
                print 'fetching icon',topic,'...'
                # cache the icon
                global dico
                if dico.has_key( int(topic) ):
                    url = dico[ int(topic) ]
                    ico = download.httpfetch(url,needlogin=False)
                    cpath = self.path+'/iconcache/'+str(topic)[:2]+'/'+str(topic)[2:4]+'/'+str(topic)+'.jpg'
                    self.ensure_dir(cpath)
                    if len(ico)>0:
                        open(cpath,'wb').write(ico)
                else:
                    cpath = self.path+'/iconcache/'+str(topic)[:2]+'/'+str(topic)[2:4]+'/'+str(topic)+'.jpg'
                    if not (os.path.exists(cpath) and len(open(cpath,'rb').read())>5):
                        c = conn.cursor()
                        c.execute( 'select content from verycd where verycdid=?',(int(topic),) )
                        content = c.fetchone()[0]
                        c.close()
                        imgurl = re.compile(r'<img src="(.*?)"',re.DOTALL).findall(content)
                        if imgurl:
                            imgurl = imgurl[0]
                            img = download.httpfetch(imgurl)
                            import Image,StringIO
                            im = Image.open( StringIO.StringIO(img) )
                            im = im.resize( (100,100) )
                            self.ensure_dir(cpath)
                            im.save(cpath)
            except Exception as what:
                print what
                conn.commit()
            self.q.task_done()

    def runthread(self,n=3):
        if not self.running:
            self.running = True
            for i in range(n):
                t = Thread(target=self.thread_fetch)
                t.setDaemon(True)
                t.start()

    def fetch(self):
        global dico
        conn = sqlite3.connect(self.path+'/verycd.sqlite3.db')
        conn.text_factory = str
        conn2 = MySQLdb.connect(user='root',passwd='guess8',db='simplecd')
        try:
            timesec = int(time.time())
            print timesec

            #what?
            if timesec%300<60:
                self.runthread()
                print 'check updates...'
                c = conn.cursor()
                timenow = time.strftime("%Y/%m/%d %H:%M:%S" ,time.gmtime(time.time()+3600*8))
                time1 = time.strftime("%Y/%m/%d %H:%M:%S" ,time.gmtime(time.time()+3600*8-4800))
                c.execute('select updtime from verycd where status!="自定义 " order by updtime desc')
                last_updtime = c.fetchone()[0]
                last_updtime = min(time1,last_updtime)
                c.close()

                url = 'http://www.verycd.com/sto/page1'
                updts = download.httpfetch(url,needlogin=False)
                idx = updts
                ids = re.compile(r'/topics/(\d+).*?<span class="date-time">(.*?)</span>.*?<span class="date-time">(.*?)</span>',re.DOTALL).findall(idx)
                updts = re.compile(r'"topic-list"(.*?)"pnav"',re.DOTALL).findall(updts)
                if updts:
                    updts = updts[0]
                    icons = re.compile(r'/topics/(\d+).*?<img src="(http://image.*?)".*?</li>',re.DOTALL).findall(updts)
                    for x in icons:
                        dico[ int(x[0]) ] = x[1]
                ids = sorted(list(set(ids)),cmp=lambda x,y:cmp(x[2],y[2]))
                c = conn.cursor()
                for id,pubtime,updtime in ids:
                    if last_updtime < updtime:
                        c.execute("select updtime from verycd where verycdid=?",(id,))
                        lupdtime = c.fetchone()
                        if lupdtime and lupdtime[0]>=updtime:
                            print "not updating",id,updtime
                            continue
                        print "updating",id
                        self.q.put([id,updtime])
                        if lupdtime:
                            open("/tmp/simplecfetch.log","a").write("%s,%s,%s>%s\n"%(timenow,id,updtime,lupdtime[0]))
                        else:
                            open("/tmp/simplecfetch.log","a").write("%s,%s,%s\n"%(timenow,id,updtime))

            # update1,disabled
            if timesec%3888<0:
                self.runthread()
                download.httpfetch('http://www.verycd.com',needlogin=False)
                url = 'http://www.verycd.com/sto/page1'
                updts = download.httpfetch(url,needlogin=False)
                updts = re.compile(r'"topic-list"(.*?)"pnav"',re.DOTALL).findall(updts)
                if updts:
                    updts = updts[0]
                    icons = re.compile(r'/topics/(\d+).*?<img src="(http://image.*?)".*?</li>',re.DOTALL).findall(updts)
                    for x in icons:
                        dico[ int(x[0]) ] = x[1]
                        self.q.put([int(x[0]),None])

            if True:
                self.runthread()
                download.httpfetch('http://www.verycd.com',needlogin=False)
                timenow = time.strftime("%Y/%m/%d %H:%M:%S" ,time.gmtime(time.time()+3600*8))
                ids=open(path+"/tofetch").read().split("\n")
                c = conn.cursor()
                for x in set(ids):
                    try:
                        id,updtime = x.split(',')
                    except:
                        continue
                    id = int(id)
                    c.execute("select updtime from verycd where verycdid=?",(id,))
                    lupdtime = c.fetchone()
                    if lupdtime and lupdtime[0]>=updtime:
                        print "not updating",id,updtime
                        continue
                    print "updating",id
                    self.q.put([id,updtime])
                    if lupdtime:
                        open("/tmp/simplecfetch.log","a").write("%s,%s,%s>%s\n"%(timenow,id,updtime,lupdtime[0]))
                    else:
                        open("/tmp/simplecfetch.log","a").write("%s,%s,%s\n"%(timenow,id,updtime))
                #open(path+"/tofetch","w").write("")

        except Exception as what:
            print what
        self.q.join()
        

if __name__ == "__main__":
    path = os.path.dirname(os.path.realpath(sys.argv[0]))
    manager = FetchManger(path=path)
    manager.fetch()