Skip to content

add modelscope tool #258

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
zzhangpurdue merged 4 commits into from
Jan 11, 2024
Merged

add modelscope tool #258

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
689f028
add modelscope tool
mushenL Jan 10, 2024
55ea2d9
add modelscope tool
mushenL Jan 10, 2024
1973dd6
add modelscope tool
mushenL Jan 11, 2024
d9176a8
Merge branch 'refactor0103' into refactor0103
zzhangpurdue Jan 11, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions modelscope_agent/tools/modelscope_tools/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
from .text_to_video_tool import TextToVideoTool
from .text_to_speech_tool import TexttoSpeechTool
from .text_address_tool import TextAddressTool
from .text_ner_tool import TextNerTool
from .text_ie_tool import TextInfoExtractTool
from .translation_en2zh_tool import TranslationEn2ZhTool
from .translation_zh2en_tool import TranslationZh2EnTool
from .image_chat_tool import ImageChatTool
31 changes: 16 additions & 15 deletions modelscope_agent/tools/modelscope_tools/image_chat_tool.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
from modelscope.utils.constant import Tasks
from typing import Union
from .pipeline_tool import ModelscopePipelineTool


Expand All @@ -9,18 +9,24 @@ class ImageChatTool(ModelscopePipelineTool):
parameters: list = [{
'name': 'image',
'description': '用户输入的图片',
'required': True
'required': True,
'type': 'string'
}, {
'name': 'text',
'description': '用户输入的文本',
'required': True
'required': True,
'type': 'string'
}]
task = Tasks.multimodal_dialogue

def construct_image_chat_input(self, **kwargs):
image = kwargs.pop('image', '')
text = kwargs.pop('text', '')

def call(self, params: str, **kwargs) -> str:
result = super().call(params, **kwargs)
image_chat = result['Data']['text']
return image_chat

def _verify_args(self, params: str) -> Union[str, dict]:
params = super()._verify_args(params)
image = params.pop('image', '')
text = params.pop('text', '')
system_prompt_1 = 'The following is a conversation between a curious human and AI assistant.'
system_prompt_2 = "The assistant gives helpful, detailed, and polite answers to the user's questions."
messages = {
Expand All @@ -41,11 +47,6 @@ def construct_image_chat_input(self, **kwargs):
},
]
}
return messages

def _remote_parse_input(self, *args, **kwargs):
messages = self.construct_image_chat_input(**kwargs)
return {'input': messages}
params = {'input': messages}
return params

def _local_parse_input(self, *args, **kwargs):
return (self.construct_image_chat_input(**kwargs)), {}
17 changes: 9 additions & 8 deletions modelscope_agent/tools/modelscope_tools/text_address_tool.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
from modelscope.utils.constant import Tasks
from .pipeline_tool import ModelscopePipelineTool


Expand All @@ -9,12 +8,14 @@ class TextAddressTool(ModelscopePipelineTool):
parameters: list = [{
'name': 'input',
'description': '用户输入的地址信息',
'required': True
'required': True,
'type': 'string'
}]
task = Tasks.token_classification

def _parse_output(self, origin_result, *args, **kwargs):
final_result = {}
for e in origin_result['output']:
final_result[e['type']] = e['span']
return {'result': final_result}
def call(self, params: str, **kwargs) -> str:
result = super().call(params, **kwargs)
address = {}
for e in result['Data']['output']:
address[e['type']] = e['span']
return address

31 changes: 17 additions & 14 deletions modelscope_agent/tools/modelscope_tools/text_ie_tool.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from typing import Union
from collections import defaultdict

from modelscope.utils.constant import Tasks
from .pipeline_tool import ModelscopePipelineTool


Expand All @@ -11,22 +11,25 @@ class TextInfoExtractTool(ModelscopePipelineTool):
parameters: list = [{
'name': 'input',
'description': '用户输入的文本',
'required': True
'required': True,
'type': 'string'
}, {
'name': 'schema',
'description': '要抽取信息的json表示',
'required': True
'required': True,
'type': 'dict'
}]
task = Tasks.siamese_uie

def _remote_parse_input(self, *args, **kwargs):
kwargs['parameters'] = {'schema': kwargs['schema']}
kwargs.pop('schema')
return kwargs
def call(self, params: str, **kwargs) -> str:
result = super().call(params, **kwargs)
InfoExtract = defaultdict(list)
for e in result['Data']['output']:
InfoExtract[e[0]['type']].append(e[0]['span'])
return str(dict(InfoExtract))

def _verify_args(self, params: str) -> Union[str, dict]:
params = super()._verify_args(params)
params['parameters'] = {'schema': params['schema']}
params.pop('schema')
return params

def _parse_output(self, origin_result, *args, **kwargs):
final_result = defaultdict(list)
for e in origin_result['output']:
final_result[e[0]['type']].append(e[0]['span'])

return {'result': dict(final_result)}
19 changes: 10 additions & 9 deletions modelscope_agent/tools/modelscope_tools/text_ner_tool.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,22 +1,23 @@
from collections import defaultdict

from modelscope.utils.constant import Tasks
from .pipeline_tool import ModelscopePipelineTool


class TextNerTool(ModelscopePipelineTool):
default_model = 'damo/nlp_raner_named-entity-recognition_chinese-base-news'
default_model = 'damo/nlp_raner_named-entity-recognition_chinese-base-cmeee'
description = '命名实体识别服务，针对需要识别的中文文本，找出其中的实体，返回json格式结果'
name = 'text-ner'
parameters: list = [{
'name': 'input',
'description': '用户输入的文本',
'required': True
'required': True,
'type': 'string'
}]
task = Tasks.named_entity_recognition

def _parse_output(self, origin_result, *args, **kwargs):
final_result = defaultdict(list)
for e in origin_result['output']:
final_result[e['type']].append(e['span'])
return {'result': dict(final_result)}
def call(self, params: str, **kwargs) -> str:
result = super().call(params, **kwargs)
ner = defaultdict(list)
for e in result['Data']['output']:
ner[e['type']].append(e['span'])
return str(dict(ner))

39 changes: 12 additions & 27 deletions modelscope_agent/tools/modelscope_tools/text_to_video_tool.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,40 +1,25 @@
import os
import tempfile
import uuid

from typing import Union
from modelscope_agent.tools.utils.output_wrapper import VideoWrapper

from modelscope.utils.constant import Tasks
from .pipeline_tool import ModelscopePipelineTool


class TextToVideoTool(ModelscopePipelineTool):
default_model = 'damo/text-to-video-synthesis'
description = '视频生成服务，针对英文文本输入，生成一段描述视频；如果是中文输入同时依赖插件modelscope_text-translation-zh2en翻译成英文'

name = 'video-generation'
parameters: list = [{
'name': 'text',
'description': '用户输入的文本信息',
'required': True
'required': True,
'type': 'string'
}]
task = Tasks.text_to_video_synthesis

def _remote_parse_input(self, *args, **kwargs):
return {'input': {'text': kwargs['text']}}

def _local_parse_input(self, *args, **kwargs):

text = kwargs.pop('text', '')
directory = tempfile.mkdtemp()
file_path = os.path.join(directory, str(uuid.uuid4()) + '.mp4')

parsed_args = ({'text': text}, )
parsed_kwargs = {'output_video': file_path}

return parsed_args, parsed_kwargs

def _parse_output(self, origin_result, remote=True):
def call(self, params: str, **kwargs) -> str:
result = super().call(params, **kwargs)
video = result['Data']['output_video']
return VideoWrapper(video)

def _verify_args(self, params: str) -> Union[str, dict]:
params = super()._verify_args(params)
params = {'input': {'text': params['text']}}
return params

video = origin_result['output_video']
return {'result': VideoWrapper(video)}
14 changes: 9 additions & 5 deletions modelscope_agent/tools/modelscope_tools/translation_en2zh_tool.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,17 +1,21 @@
from modelscope.utils.constant import Tasks
from .pipeline_tool import ModelscopePipelineTool


class TranslationEn2ZhTool(ModelscopePipelineTool):
default_model = 'damo/nlp_csanmt_translation_en2zh'
description = '根据输入指令，将相应的英文文本翻译成中文回复'
name = 'text-translation-en2zh'
task = Tasks.translation
parameters: list = [{
'name': 'input',
'description': '用户输入的英文文本',
'required': True
'required': True,
'type': 'string'
}]

def _parse_output(self, origin_result, *args, **kwargs):
return {'result': origin_result['translation']}
def call(self, params: str, **kwargs) -> str:
result = super().call(params, **kwargs)
zh = result['Data']['translation']
return zh

# def _parse_output(self, origin_result, *args, **kwargs):
# return {'result': origin_result['translation']}
14 changes: 9 additions & 5 deletions modelscope_agent/tools/modelscope_tools/translation_zh2en_tool.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,17 +1,21 @@
from modelscope.utils.constant import Tasks
from .pipeline_tool import ModelscopePipelineTool


class TranslationZh2EnTool(ModelscopePipelineTool):
default_model = 'damo/nlp_csanmt_translation_zh2en'
description = '根据输入指令，将相应的中文文本翻译成英文回复'
name = 'text-translation-zh2en'
task = Tasks.translation
parameters: list = [{
'name': 'input',
'description': '用户输入的中文文本',
'required': True
'required': True,
'type': 'string'
}]

def _parse_output(self, origin_result, *args, **kwargs):
return {'result': origin_result['translation']}
def call(self, params: str, **kwargs) -> str:
result = super().call(params, **kwargs)
en = result['Data']['translation']
return en

# def _parse_output(self, origin_result, *args, **kwargs):
# return {'result': origin_result['translation']}
1 change: 1 addition & 0 deletions modelscope_agent/tools/utils/output_wrapper.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -216,3 +216,4 @@ def get_raw_output(exec_result: Dict):
# display(Pretty(exec_result))
#
# return

58 changes: 57 additions & 1 deletion tests/tools/test_pipeline_tool.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,67 @@


def test_modelscope_speech_generation():
from modelscope_agent.tools import TexttoSpeechTool
from modelscope_agent.tools.modelscope_tools import TexttoSpeechTool
kwargs = """{'input': '北京今天天气怎样?', 'gender': 'man'}"""
txt2speech = TexttoSpeechTool(cfg)
res = txt2speech.call(kwargs)
print(res)

def test_modelscope_video_generation():
from modelscope_agent.tools.modelscope_tools import TextToVideoTool
kwargs = """{'text': '一个正在打篮球的人'}"""
txt2video = TextToVideoTool(cfg)
res = txt2video.call(kwargs)
print(res)

def test_modelscope_text_address():
from modelscope_agent.tools.modelscope_tools import TextAddressTool
kwargs = """{'input': '北京朝阳望京东金辉大厦'}"""
txt_addr = TextAddressTool(cfg)
res = txt_addr.call(kwargs)
print(res)

def test_modelscope_text_ner():
from modelscope_agent.tools.modelscope_tools import TextNerTool
kwargs = """{'input': '多数新生儿甲亢在出生时即有症状，表现为突眼、甲状腺肿大、烦躁、多动、心动过速、呼吸急促，严重可出现心力衰竭，血T3、T4升高，TSH下降。'}"""
txt_ner = TextNerTool(cfg)
res = txt_ner.call(kwargs)
print(res)

def test_modelscope_text_ie():
from modelscope_agent.tools.modelscope_tools import TextInfoExtractTool
kwargs = """{'input': '很满意，音质很好，发货速度快，值得购买', 'schema': {'属性词': {'情感词': null}}}"""
txt_ie = TextInfoExtractTool(cfg)
res = txt_ie.call(kwargs)
print(res)

def test_modelscope_en2zh():
from modelscope_agent.tools.modelscope_tools import TranslationEn2ZhTool
kwargs = """{'input': 'Autonomous agents have long been a prominent research focus in both academic and industry communities.'}"""
zh_txt = TranslationEn2ZhTool(cfg)
res = zh_txt.call(kwargs)
print(res)

def test_modelscope_zh2en():
from modelscope_agent.tools.modelscope_tools import TranslationZh2EnTool
kwargs = """{'input': '北京今天天气怎样?'}"""
en_txt = TranslationZh2EnTool(cfg)
res = en_txt.call(kwargs)
print(res)

def test_modelscope_image_chat():
from modelscope_agent.tools.modelscope_tools import ImageChatTool
kwargs = """{'image': 'http://mm-chatgpt.oss-cn-zhangjiakou.aliyuncs.com/mplug_owl_demo/released_checkpoint/portrait_input.png', 'text': 'Describe the facial expression of the man.'}"""
image_chat = ImageChatTool(cfg)
res = image_chat.call(kwargs)
print(res)


test_modelscope_speech_generation()
test_modelscope_video_generation()
test_modelscope_text_address()
test_modelscope_text_ner()
test_modelscope_text_ie()
test_modelscope_en2zh()
test_modelscope_zh2en()
test_modelscope_image_chat()