binux /
pyspider
| @@ 25-55 (lines=31) @@ | ||
| 22 | class TaskDB(SplitTableMixin, BaseTaskDB): |
|
| 23 | __tablename__ = '' |
|
| 24 | ||
| 25 | def __init__(self, url): |
|
| 26 | self.table = Table('__tablename__', MetaData(), |
|
| 27 | Column('taskid', String(64), primary_key=True, nullable=False), |
|
| 28 | Column('project', String(64)), |
|
| 29 | Column('url', String(1024)), |
|
| 30 | Column('status', Integer), |
|
| 31 | Column('schedule', LargeBinary), |
|
| 32 | Column('fetch', LargeBinary), |
|
| 33 | Column('process', LargeBinary), |
|
| 34 | Column('track', LargeBinary), |
|
| 35 | Column('lastcrawltime', Float(32)), |
|
| 36 | Column('updatetime', Float(32)), |
|
| 37 | mysql_engine='InnoDB', |
|
| 38 | mysql_charset='utf8' |
|
| 39 | ) |
|
| 40 | ||
| 41 | self.url = make_url(url) |
|
| 42 | if self.url.database: |
|
| 43 | database = self.url.database |
|
| 44 | self.url.database = None |
|
| 45 | try: |
|
| 46 | engine = create_engine(self.url) |
|
| 47 | conn = engine.connect() |
|
| 48 | conn.execute("commit") |
|
| 49 | conn.execute("CREATE DATABASE %s" % database) |
|
| 50 | except sqlalchemy.exc.SQLAlchemyError: |
|
| 51 | pass |
|
| 52 | self.url.database = database |
|
| 53 | self.engine = create_engine(url) |
|
| 54 | ||
| 55 | self._list_project() |
|
| 56 | ||
| 57 | def _create_project(self, project): |
|
| 58 | assert re.match(r'^\w+$', project) is not None |
|
| @@ 22-49 (lines=28) @@ | ||
| 19 | class ProjectDB(BaseProjectDB): |
|
| 20 | __tablename__ = 'projectdb' |
|
| 21 | ||
| 22 | def __init__(self, url): |
|
| 23 | self.table = Table(self.__tablename__, MetaData(), |
|
| 24 | Column('name', String(64)), |
|
| 25 | Column('group', String(64)), |
|
| 26 | Column('status', String(16)), |
|
| 27 | Column('script', Text), |
|
| 28 | Column('comments', String(1024)), |
|
| 29 | Column('rate', Float(11)), |
|
| 30 | Column('burst', Float(11)), |
|
| 31 | Column('updatetime', Float(32)), |
|
| 32 | mysql_engine='InnoDB', |
|
| 33 | mysql_charset='utf8' |
|
| 34 | ) |
|
| 35 | ||
| 36 | self.url = make_url(url) |
|
| 37 | if self.url.database: |
|
| 38 | database = self.url.database |
|
| 39 | self.url.database = None |
|
| 40 | try: |
|
| 41 | engine = create_engine(self.url) |
|
| 42 | conn = engine.connect() |
|
| 43 | conn.execute("commit") |
|
| 44 | conn.execute("CREATE DATABASE %s" % database) |
|
| 45 | except sqlalchemy.exc.SQLAlchemyError: |
|
| 46 | pass |
|
| 47 | self.url.database = database |
|
| 48 | self.engine = create_engine(url) |
|
| 49 | self.table.create(self.engine, checkfirst=True) |
|
| 50 | ||
| 51 | @staticmethod |
|
| 52 | def _parse(data): |
|
