Spaces:
Running
on
Zero
Running
on
Zero
| import re,json,os | |
| import pandas as pd | |
| from rag.nlp import rag_tokenizer | |
| from . import regions | |
| current_file_path = os.path.dirname(os.path.abspath(__file__)) | |
| GOODS = pd.read_csv(os.path.join(current_file_path, "res/corp_baike_len.csv"), sep="\t", header=0).fillna(0) | |
| GOODS["cid"] = GOODS["cid"].astype(str) | |
| GOODS = GOODS.set_index(["cid"]) | |
| CORP_TKS = json.load(open(os.path.join(current_file_path, "res/corp.tks.freq.json"), "r")) | |
| GOOD_CORP = json.load(open(os.path.join(current_file_path, "res/good_corp.json"), "r")) | |
| CORP_TAG = json.load(open(os.path.join(current_file_path, "res/corp_tag.json"), "r")) | |
| def baike(cid, default_v=0): | |
| global GOODS | |
| try: | |
| return GOODS.loc[str(cid), "len"] | |
| except Exception as e: | |
| pass | |
| return default_v | |
| def corpNorm(nm, add_region=True): | |
| global CORP_TKS | |
| if not nm or type(nm)!=type(""):return "" | |
| nm = rag_tokenizer.tradi2simp(rag_tokenizer.strQ2B(nm)).lower() | |
| nm = re.sub(r"&", "&", nm) | |
| nm = re.sub(r"[\(\)（）\+'\"\t \*\\【】-]+", " ", nm) | |
| nm = re.sub(r"([—-]+.*| +co\..*|corp\..*| +inc\..*| +ltd.*)", "", nm, 10000, re.IGNORECASE) | |
| nm = re.sub(r"(计算机|技术|(技术|科技|网络)*有限公司|公司|有限|研发中心|中国|总部)$", "", nm, 10000, re.IGNORECASE) | |
| if not nm or (len(nm)<5 and not regions.isName(nm[0:2])):return nm | |
| tks = rag_tokenizer.tokenize(nm).split(" ") | |
| reg = [t for i,t in enumerate(tks) if regions.isName(t) and (t != "中国" or i > 0)] | |
| nm = "" | |
| for t in tks: | |
| if regions.isName(t) or t in CORP_TKS:continue | |
| if re.match(r"[0-9a-zA-Z\\,.]+", t) and re.match(r".*[0-9a-zA-Z\,.]+$", nm):nm += " " | |
| nm += t | |
| r = re.search(r"^([^a-z0-9 \(\)&]{2,})[a-z ]{4,}$", nm.strip()) | |
| if r:nm = r.group(1) | |
| r = re.search(r"^([a-z ]{3,})[^a-z0-9 \(\)&]{2,}$", nm.strip()) | |
| if r:nm = r.group(1) | |
| return nm.strip() + (("" if not reg else "(%s)"%reg[0]) if add_region else "") | |
| def rmNoise(n): | |
| n = re.sub(r"[\(（][^()（）]+[)）]", "", n) | |
| n = re.sub(r"[,. &（）()]+", "", n) | |
| return n | |
| GOOD_CORP = set([corpNorm(rmNoise(c), False) for c in GOOD_CORP]) | |
| for c,v in CORP_TAG.items(): | |
| cc = corpNorm(rmNoise(c), False) | |
| if not cc: print (c) | |
| CORP_TAG = {corpNorm(rmNoise(c), False):v for c,v in CORP_TAG.items()} | |
| def is_good(nm): | |
| global GOOD_CORP | |
| if nm.find("外派")>=0:return False | |
| nm = rmNoise(nm) | |
| nm = corpNorm(nm, False) | |
| for n in GOOD_CORP: | |
| if re.match(r"[0-9a-zA-Z]+$", n): | |
| if n == nm: return True | |
| elif nm.find(n)>=0:return True | |
| return False | |
| def corp_tag(nm): | |
| global CORP_TAG | |
| nm = rmNoise(nm) | |
| nm = corpNorm(nm, False) | |
| for n in CORP_TAG.keys(): | |
| if re.match(r"[0-9a-zA-Z., ]+$", n): | |
| if n == nm: return CORP_TAG[n] | |
| elif nm.find(n)>=0: | |
| if len(n)<3 and len(nm)/len(n)>=2:continue | |
| return CORP_TAG[n] | |
| return [] | |