discours.io/core
5
0
Fork 0
Code Pull Requests Actions Projects Activity

Merge branch 'ackee-test' into 'main'

Browse Source 
Page views

See merge request discoursio/discoursio-api!6
This commit is contained in:
Igor 2023-01-18 21:35:49 +00:00
commit 84b1f37d48
16 changed files with 369 additions and 136 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
migration/__init__.py
View File

@ -10,7 +10,8 @@ from migration.tables.comments import migrate_2stage as migrateComment_2stage
from migration.tables.content_items import get_shout_slug
from migration.tables.content_items import migrate as migrateShout
from migration.tables.topics import migrate as migrateTopic
from migration.tables.users import migrate as migrateUser
from migration.tables.users import migrate as migrateUser, post_migrate as users_post_migrate
from migration.tables.remarks import migrate as migrateRemark
from migration.tables.users import migrate_2stage as migrateUser_2stage
from orm.reaction import Reaction
from orm import init_tables
@ -41,6 +42,7 @@ async def users_handle(storage):
ce = 0
for entry in storage["users"]["data"]:
ce += migrateUser_2stage(entry, id_map)
users_post_migrate()
async def topics_handle(storage):
@ -135,6 +137,15 @@ async def shouts_handle(storage, args):
print("[migration] " + str(anonymous_author) + " authored by @anonymous")
async def remarks_handle(storage):
print("[migration] comments")
c = 0
for entry_remark in storage["remarks"]["data"]:
remark = await migrateRemark(entry_remark, storage)
c += 1
print("[migration] " + str(c) + " remarks migrated")
async def comments_handle(storage):
print("[migration] comments")
id_map = {}
@ -170,6 +181,8 @@ async def all_handle(storage, args):
await topics_handle(storage)
print("[migration] users and topics are migrated")
await shouts_handle(storage, args)
# print("[migration] remarks...")
# await remarks_handle(storage)
print("[migration] migrating comments")
await comments_handle(storage)
# export_email_subscriptions()
@ -190,6 +203,7 @@ def data_load():
"cats": [],
"tags": [],
},
"remarks": {"data": []},
"users": {"by_oid": {}, "by_slug": {}, "data": []},
"replacements": json.loads(open("migration/tables/replacements.json").read()),
}
@ -210,6 +224,11 @@ def data_load():
content_data = json.loads(open("migration/data/content_items.json").read())
storage["shouts"]["data"] = content_data
print("[migration.load] " + str(len(content_data)) + " content items ")
remarks_data = json.loads(open("migration/data/remarks.json").read())
storage["remarks"]["data"] = remarks_data
print("[migration.load] " + str(len(remarks_data)) + " remarks data ")
# fill out storage
for x in users_data:
storage["users"]["by_oid"][x["_id"]] = x

2
migration/tables/content_items.py
View File

@ -176,7 +176,7 @@ async def migrate(entry, storage):
await content_ratings_to_reactions(entry, shout_dict["slug"])
# shout views
await ViewedStorage.increment(shout_dict["slug"], amount=entry.get("views", 1))
await ViewedStorage.increment(shout_dict["slug"], amount=entry.get("views", 1), viewer='old-discours')
# del shout_dict['ratings']
storage["shouts"]["by_oid"][entry["_id"]] = shout_dict

31
migration/tables/remarks.py Normal file
View File

@ -0,0 +1,31 @@
from base.orm import local_session
from migration.extract import extract_md
from migration.html2text import html2text
from orm.remark import Remark
def migrate(entry, storage):
post_oid = entry['contentItem']
print(post_oid)
shout_dict = storage['shouts']['by_oid'].get(post_oid)
remark = {
"shout": shout_dict['id'],
"body": extract_md(
html2text(entry['body']),
entry['_id']
),
"desc": extract_md(
html2text(
entry['textAfter'] or '' + \
entry['textBefore'] or '' + \
entry['textSelected'] or ''
),
entry["_id"]
)
}
with local_session() as session:
rmrk = Remark.create(**remark)
session.commit()
del rmrk["_sa_instance_state"]
return rmrk

20
migration/tables/users.py
View File

@ -1,7 +1,9 @@
import re
from bs4 import BeautifulSoup
from dateutil.parser import parse
from sqlalchemy.exc import IntegrityError
from bs4 import BeautifulSoup
import re
from base.orm import local_session
from orm.user import AuthorFollower, User, UserRating
@ -108,6 +110,20 @@ def migrate(entry):
return user_dict
def post_migrate():
old_discours_dict = {
"slug": "old-discours",
"username": "old-discours",
"email": "old@discours.io",
"name": "Просмотры на старой версии сайта"
}
with local_session() as session:
old_discours_user = User.create(**old_discours_dict)
session.add(old_discours_user)
session.commit()
def migrate_2stage(entry, id_map):
ce = 0
for rating_entry in entry.get("ratings", []):

27
orm/collab.py
View File

@ -1,27 +0,0 @@
from datetime import datetime
from sqlalchemy import Column, ForeignKey, DateTime, String
from sqlalchemy.orm import relationship
from base.orm import Base
from orm.user import User
class CollabAuthor(Base):
__tablename__ = "collab_author"
id = None # type: ignore
collab = Column(ForeignKey("collab.id"), primary_key=True)
author = Column(ForeignKey("user.id"), primary_key=True)
# accepted = Column(Boolean, default=False)
class Collab(Base):
__tablename__ = "collab"
title = Column(String, nullable=True, comment="Title")
body = Column(String, nullable=True, comment="Body")
pic = Column(String, nullable=True, comment="Picture")
authors = relationship(lambda: User, secondary=CollabAuthor.__tablename__)
# invites = relationship(lambda: User, secondary=CollabInvited.__tablename__)
createdAt = Column(DateTime, default=datetime.now, comment="Created At")
chat = Column(String, unique=True, nullable=False)

40
orm/draft.py Normal file
View File

@ -0,0 +1,40 @@
from datetime import datetime
from sqlalchemy import Boolean, Column, ForeignKey, DateTime, String
from sqlalchemy.orm import relationship
from base.orm import Base
from orm.user import User
from orm.topic import Topic
class DraftTopic(Base):
__tablename__ = "draft_topic"
id = None # type: ignore
collab = Column(ForeignKey("draft_collab.id"), primary_key=True)
topic = Column(ForeignKey("topic.id"), primary_key=True)
class DraftAuthor(Base):
__tablename__ = "draft_author"
id = None # type: ignore
collab = Column(ForeignKey("draft_collab.id"), primary_key=True)
author = Column(ForeignKey("user.id"), primary_key=True)
accepted = Column(Boolean, default=False)
class DraftCollab(Base):
__tablename__ = "draft_collab"
slug = Column(String, nullable=True, comment="Slug")
title = Column(String, nullable=True, comment="Title")
subtitle = Column(String, nullable=True, comment="Subtitle")
layout = Column(String, nullable=True, comment="Layout format")
body = Column(String, nullable=True, comment="Body")
cover = Column(String, nullable=True, comment="Cover")
authors = relationship(lambda: User, secondary=DraftAuthor.__tablename__)
topics = relationship(lambda: Topic, secondary=DraftTopic.__tablename__)
createdAt = Column(DateTime, default=datetime.now, comment="Created At")
updatedAt = Column(DateTime, default=datetime.now, comment="Updated At")
chat = Column(String, unique=True, nullable=True)

15
orm/remark.py Normal file
View File

@ -0,0 +1,15 @@
from datetime import datetime
from enum import Enum as Enumeration
from sqlalchemy import Column, DateTime, Enum, ForeignKey, String
from base.orm import Base
class Remark(Base):
__tablename__ = "remark"
body = Column(String, nullable=False)
desc = Column(String, default='')
shout = Column(ForeignKey("shout.id"), nullable=True, index=True, comment="Shout")

2
orm/shout.py
View File

@ -50,7 +50,7 @@ class Shout(Base):
subtitle = Column(String, nullable=True)
layout = Column(String, nullable=True)
mainTopic = Column(ForeignKey("topic.slug"), nullable=True)
cover = Column(String, nullable=True)
cover = Column(String, nullable=True, comment="Cover")
authors = relationship(lambda: User, secondary=ShoutAuthor.__tablename__)
topics = relationship(lambda: Topic, secondary=ShoutTopic.__tablename__)
reactions = relationship(lambda: Reaction)

9
resolvers/__init__.py
View File

@ -8,7 +8,8 @@ from resolvers.auth import (
get_current_user,
)
from resolvers.create.collab import remove_coauthor, invite_coauthor
from resolvers.create.collab import load_drafts, create_draft, update_draft, delete_draft,\
accept_coauthor, invite_coauthor
from resolvers.create.migrate import markdown_body
from resolvers.create.editor import create_shout, delete_shout, update_shout
@ -93,8 +94,12 @@ __all__ = [
# create.migrate
"markdown_body",
# create.collab
"load_drafts",
"create_draft",
"update_draft",
"delete_draft",
"invite_coauthor",
"remove_coauthor",
"accept_coauthor",
# zine.topics
"topics_all",
"topics_by_community",

152
resolvers/create/collab.py
View File

@ -3,84 +3,130 @@ from auth.credentials import AuthCredentials
from base.orm import local_session
from base.resolvers import query, mutation
from base.exceptions import ObjectNotExist, BaseHttpException
from orm.collab import Collab, CollabAuthor
from orm.draft import DraftCollab, DraftAuthor, DraftTopic
from orm.shout import Shout
from orm.user import User
@query.field("getCollabs")
@query.field("loadDrafts")
@login_required
async def get_collabs(_, info):
async def load_drafts(_, info):
auth: AuthCredentials = info.context["request"].auth
drafts = []
with local_session() as session:
drafts = session.query(DraftCollab).filter(auth.user_id in DraftCollab.authors)
return drafts
@mutation.field("createDraft") # TODO
@login_required
async def create_draft(_, info, draft_input):
auth: AuthCredentials = info.context["request"].auth
with local_session() as session:
collabs = session.query(Collab).filter(auth.user_id in Collab.authors)
return collabs
collab = DraftCollab.create(**draft_input)
session.add(collab)
session.commit()
@mutation.field("inviteCoauthor")
@login_required
async def invite_coauthor(_, info, author: int = 0, shout: int = 0):
auth: AuthCredentials = info.context["request"].auth
with local_session() as session:
s = session.query(Shout).where(Shout.id == shout).one()
if not s:
raise ObjectNotExist("invalid shout id")
else:
c = session.query(Collab).where(Collab.shout == shout).one()
if auth.user_id not in c.authors:
raise BaseHttpException("you are not in authors list")
else:
invited_user = session.query(User).where(User.id == author).one()
c.invites.append(invited_user)
session.add(c)
session.commit()
# TODO: email notify
# TODO: email notify to all authors
return {}
@mutation.field("removeCoauthor")
@mutation.field("deleteDraft") # TODO
@login_required
async def remove_coauthor(_, info, author: int = 0, shout: int = 0):
async def delete_draft(_, info, draft: int = 0):
auth: AuthCredentials = info.context["request"].auth
with local_session() as session:
s = session.query(Shout).where(Shout.id == shout).one() # raises Error when not found
collab = session.query(DraftCollab).where(DraftCollab.id == draft_input.id).one()
if auth.user_id not in s.authors:
raise BaseHttpException("only owner can remove coauthors")
# raise BaseHttpException("only owner can remove coauthors")
return {
"error": "Only authors can update a draft"
}
elif not collab:
return {
"error": "There is no draft with this id"
}
else:
c = session.query(Collab).where(Collab.shout == shout).one()
ca = session.query(CollabAuthor).join(User).where(c.shout == shout, User.id == author).one()
session.remve(ca)
c.invites = filter(lambda x: x.id == author, c.invites)
c.authors = filter(lambda x: x.id == author, c.authors)
session.add(c)
session.delete(collab)
session.commit()
return {}
@mutation.field("updateDraft") # TODO: draft input type
@login_required
async def update_draft(_, info, draft_input):
auth: AuthCredentials = info.context["request"].auth
with local_session() as session:
collab = session.query(DraftCollab).where(DraftCollab.id == draft_input.id).one() # raises Error when not found
if auth.user_id not in s.authors:
# raise BaseHttpException("only owner can remove coauthors")
return {
"error": "Only authors can update draft"
}
elif not s:
return {
"error": "There is no draft with this id"
}
else:
draft_input["updatedAt"] = datetime.now(tz=timezone.utc)
collab.update(draft_input)
session.commit()
# TODO: email notify
return {}
@mutation.field("acceptCoauthor")
@mutation.field("inviteAuthor")
@login_required
async def accept_coauthor(_, info, shout: int):
async def invite_coauthor(_, info, author: int = 0, draft: int = 0):
auth: AuthCredentials = info.context["request"].auth
with local_session() as session:
s = session.query(Shout).where(Shout.id == shout).one()
if not s:
raise ObjectNotExist("invalid shout id")
c = session.query(DraftCollab).where(DraftCollab.id == draft).one()
if auth.user_id not in c.authors:
# raise BaseHttpException("you are not in authors list")
return {
"error": "You are not in authors list"
}
elif c.id:
invited_user = session.query(User).where(User.id == author).one()
da = DraftAuthor.create({
"accepted": False,
"collab": c.id,
"author": invited_user.id
})
session.add(da)
session.commit()
else:
c = session.query(Collab).where(Collab.shout == shout).one()
accepted = filter(lambda x: x.id == auth.user_id, c.invites).pop()
if accepted:
c.authors.append(accepted)
s.authors.append(accepted)
session.add(s)
session.add(c)
session.commit()
return {}
else:
raise BaseHttpException("only invited can accept")
return {
"error": "Draft is not found"
}
# TODO: email notify
return {}
@mutation.field("inviteAccept")
@login_required
async def accept_coauthor(_, info, draft: int):
auth: AuthCredentials = info.context["request"].auth
with local_session() as session:
# c = session.query(DraftCollab).where(DraftCollab.id == draft).one()
a = session.query(DraftAuthor).where(DraftAuthor.collab == draft).filter(DraftAuthor.author == auth.user_id).one()
if not a.accepted:
a.accepted = True
session.commit()
# TODO: email notify
return {}
elif a.accepted == True:
return {
"error": "You have accepted invite before"
}
else:
# raise BaseHttpException("only invited can accept")
return {
"error": "You don't have an invitation yet"
}

2
resolvers/create/editor.py
View File

@ -14,7 +14,7 @@ from resolvers.zine.reactions import reactions_follow, reactions_unfollow
from services.zine.gittask import GitTask
from resolvers.inbox.chats import create_chat
from services.inbox.storage import MessagesStorage
from orm.collab import Collab
from orm.draft import DraftCollab
@mutation.field("createShout")

5
resolvers/zine/profile.py
View File

@ -218,10 +218,13 @@ def author_unfollow(user_id, slug):
).first()
)
if not flw:
raise Exception("[resolvers.profile] follower not exist, cant unfollow")
return {
"error": "Follower is not exist, cant unfollow"
}
else:
session.delete(flw)
session.commit()
return {}
@query.field("authorsAll")

2
resolvers/zine/reactions.py
View File

@ -196,7 +196,7 @@ async def update_reaction(_, info, reaction={}):
if not r:
return {"error": "invalid reaction id"}
if r.createdBy != user.slug:
if r.createdBy != user.id:
return {"error": "access denied"}
r.body = reaction["body"]

48
resolvers/zine/remark.py Normal file
View File

@ -0,0 +1,48 @@
from datetime import datetime, timedelta, timezone
from sqlalchemy.orm import joinedload, aliased
from sqlalchemy.sql.expression import desc, asc, select, func
from base.orm import local_session
from base.resolvers import query, mutation
from base.exceptions import ObjectNotExist
from orm.remark import Remark
@mutation.field("createRemark")
@login_required
async def create_remark(_, info, slug, body):
auth = info.context["request"].auth
user_id = auth.user_id
with local_session() as session:
tt = Remark.create(slug=slug, body=body)
session.commit()
return
@mutation.field("updateRemark")
@login_required
async def update_remark(_, info, slug, body = ''):
auth = info.context["request"].auth
user_id = auth.user_id
with local_session() as session:
rmrk = session.query(Remark).where(Remark.slug == slug).one()
if body:
tt.body = body
session.add(rmrk)
session.commit()
return
@mutation.field("deleteRemark")
@login_required
async def delete_remark(_, info, slug):
auth = info.context["request"].auth
user_id = auth.user_id
with local_session() as session:
rmrk = session.query(Remark).where(Remark.slug == slug).one()
rmrk.remove()
session.commit()
return
@query.field("loadRemark")
@login_required
async def load_remark(_, info, slug):
pass

40
schema.graphql
View File

@ -69,6 +69,7 @@ type Result {
members: [ChatMember]
shout: Shout
shouts: [Shout]
drafts: [DraftCollab]
author: Author
authors: [Author]
reaction: Reaction
@ -128,6 +129,17 @@ input TopicInput {
# parents: [String]
}
input DraftInput {
slug: String
topics: [Int]
authors: [Int]
title: String
subtitle: String
body: String
cover: String
}
input ReactionInput {
kind: ReactionKind!
shout: Int!
@ -189,10 +201,12 @@ type Mutation {
updateReaction(reaction: ReactionInput!): Result!
deleteReaction(reaction: Int!): Result!
# collab
inviteCoauthor(author: String!, shout: Int!): Result!
removeCoauthor(author: String!, shout: Int!): Result!
acceptCoauthor(shout: Int!): Result!
# draft / collab
createDraft(draft: DraftInput!): Result!
updateDraft(draft: DraftInput!): Result!
deleteDraft(draft: Int!): Result!
inviteAccept(draft: Int!): Result!
inviteAuthor(draft: Int!, author: Int!): Result!
# following
follow(what: FollowingEntity!, slug: String!): Result!
@ -289,8 +303,8 @@ type Query {
authorsAll: [Author]!
getAuthor(slug: String!): User
# collab
getCollabs: [Collab]!
# draft/collab
loadDrafts: [DraftCollab]!
# migrate
markdownBody(body: String!): String!
@ -525,10 +539,16 @@ type Chat {
private: Boolean
}
type Collab {
authors: [String]!
invites: [String]
shout: Shout
type DraftCollab {
slug: String
title: String
subtitle: String
body: String
cover: String
layout: String
authors: [Int]!
topics: [String]
chat: Chat
createdAt: Int!
updatedAt: Int
}

89
services/stat/viewed.py
View File

@ -1,16 +1,17 @@
import asyncio
import time
from datetime import timedelta, timezone, datetime
from os import environ, path
from ssl import create_default_context
from gql import Client, gql
from gql.transport.aiohttp import AIOHTTPTransport
from base.orm import local_session
from sqlalchemy import func
from base.orm import local_session
from orm import User, Topic
from orm.shout import ShoutTopic, Shout
from orm.viewed import ViewedEntry
from ssl import create_default_context
from os import environ, path
load_facts = gql("""
query getDomains {
@ -64,7 +65,7 @@ class ViewedStorage:
views = None
pages = None
domains = None
period = 24 * 60 * 60 # one time a day
period = 60 * 60 # every hour
client = None
auth_result = None
disabled = False
@ -98,8 +99,8 @@ class ViewedStorage:
p = page["value"].split("?")[0]
slug = p.split('discours.io/')[-1]
shouts[slug] = page["count"]
for slug, v in shouts:
await ViewedStorage.increment(slug, v)
for slug in shouts.keys():
await ViewedStorage.increment(slug, shouts[slug])
except Exception:
pass
print("[stat.viewed] ⎪ %d pages collected " % len(shouts.keys()))
@ -164,15 +165,31 @@ class ViewedStorage:
self = ViewedStorage
async with self.lock:
with local_session() as session:
shout = session.query(Shout).where(Shout.slug == shout_slug).one()
viewer = session.query(User).where(User.slug == viewer).one()
# TODO: user slug -> id
viewed = session.query(
ViewedEntry
).join(
Shout
).join(
User
).filter(
User.slug == viewer,
Shout.slug == shout_slug
).first()
if viewed:
viewed.amount = amount
print("amount: %d" % amount)
else:
shout = session.query(Shout).where(Shout.slug == shout_slug).one()
viewer = session.query(User).where(User.slug == viewer).one()
new_viewed = ViewedEntry.create(**{
"viewer": viewer.id,
"shout": shout.id,
"amount": amount
})
session.add(new_viewed)
viewed = ViewedEntry.create(**{
"viewer": viewer.id,
"shout": shout.id,
"amount": amount
})
session.add(viewed)
session.commit()
self.by_shouts[shout_slug] = self.by_shouts.get(shout_slug, 0) + amount
self.update_topics(session, shout_slug)
@ -184,25 +201,25 @@ class ViewedStorage:
self = ViewedStorage
if self.disabled:
return
async with self.lock:
while True:
try:
print("[stat.viewed] - updating views...")
await self.update_pages()
failed = 0
except Exception:
failed += 1
print("[stat.viewed] - update failed #%d, wait 10 seconds" % failed)
if failed > 3:
print("[stat.viewed] - not trying to update anymore")
break
if failed == 0:
when = datetime.now(timezone.utc) + timedelta(seconds=self.period)
t = format(when.astimezone().isoformat())
print("[stat.viewed] ⎩ next update: %s" % (
t.split("T")[0] + " " + t.split("T")[1].split(".")[0]
))
await asyncio.sleep(self.period)
else:
await asyncio.sleep(10)
print("[stat.viewed] - trying to update data again")
while True:
try:
print("[stat.viewed] - updating views...")
await self.update_pages()
failed = 0
except Exception:
failed += 1
print("[stat.viewed] - update failed #%d, wait 10 seconds" % failed)
if failed > 3:
print("[stat.viewed] - not trying to update anymore")
break
if failed == 0:
when = datetime.now(timezone.utc) + timedelta(seconds=self.period)
t = format(when.astimezone().isoformat())
print("[stat.viewed] ⎩ next update: %s" % (
t.split("T")[0] + " " + t.split("T")[1].split(".")[0]
))
await asyncio.sleep(self.period)
else:
await asyncio.sleep(10)
print("[stat.viewed] - trying to update data again")