How to use FIM and Function call?

[Shuyangg]

Hi there, Nice work!

Do you support funcion call? and how should I use FIM?

Thank you

[zhujiangang]

you can put your tool information in SYSTEM or OBSERVATION role in your chat messages in order to support function call.

[fengzhu1]

@Shuyangg Our Ling models are capable of supporting function calling. Below, we provide a demo to guide you through the process. If you have any questions about this feature, please don’t hesitate to reach out to us.

Regarding FIM: Is it intended to be a callable function, or does it serve another purpose?

If you wish to utilize our function-calling capability, please follow the following example:

Supported Roles: We support four roles in chat dialogs: (1)SYSTEM (2)ASSISTANT (3)HUMAN (4)OBSERVATION

Example: An Agent with two functions, i.e., searching and booking flights. Please put the following codes in a notebook.

---

Step 1. Model Loading (Code):

from transformers import AutoModelForCausalLM, AutoTokenizer

model_name = "inclusionAI/Ling-lite"

model = AutoModelForCausalLM.from_pretrained(
    model_name,
    torch_dtype="auto",
    device_map="auto",
    trust_remote_code=True
)
tokenizer = AutoTokenizer.from_pretrained(model_name)

chat_messages = []

def model_generate(chat_messages):
    text = tokenizer.apply_chat_template(
        chat_messages,
        tokenize=False,
        add_generation_prompt=True
    )
    model_inputs = tokenizer([text], return_tensors="pt").to(model.device)

    generated_ids = model.generate(
        **model_inputs,
        max_new_tokens=512
    )
    generated_ids = [
        output_ids[len(input_ids):] for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids)
    ]
    response = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]
    return response

---

Step 2 System message and tool descriptions (Code):

import json
TOOLS = [
  {
    "type": "function",
    "function": {
      "name": "get_current_flights_by_cities_and_date",
      "description": "Get the flight information.",
      "parameters": {
        "type": "object",
        "properties": {
          "departure_city": {
            "type": "string",
            "description": "The departure city, in the format \"City\"."
          },
          "arrival_city": {
            "type": "string",
            "description": "The arrival city, in the format \"City\"."
          },
          "date": {
            "type": "string",
            "description": "the search date, in the format \"yyyy-mm-dd\"."
          }
        },
        "required": [
          "departure_city",
          "arrival_city",
          "date"
        ]
      }
    }
  },
  {
    "type": "function",
    "function": {
      "name": "book_a_flight",
      "description": "Book a flight from an airport to another airport.",
      "parameters": {
        "type": "object",
        "properties": {
          "airlines": {
            "type": "string",
            "description": "The airlines."
          },
          "flight_number":{
            "type": "string",
            "description": "The flight number."
          },
          "departure_airport": {
            "type": "string",
            "description": "The departure airport."
          },
          "arrival_airport": {
            "type": "string",
            "description": "The arrival airport."
          },
          "departure_time": {
            "type": "string",
            "description": "The departure time, in the format \"yyyy-mm-dd HH:mm:ss\"."
          },
          "arrival_time": {
            "type": "string",
            "description": "The arrival time, in the format \"yyyy-mm-dd HH:mm:ss\""
          },
          "price": {
            "type": "string",
            "description": "The price."
          }
        },
        "required": [
          "airlines",
          "flight_number",
          "departure_airport",
          "arrival_airport",
          "departure_time",
          "arrival_time",
          "price"
        ]
      }
    }
  }
]

system_message_template="""You are a LLM agent, and the currently supported tools are as follows:

{{tool_info}}

Please determine on your own whether a tool needs to be invoked to address the user’s current problem. If a tool is required, respond in the following format without adding any comments (//):
***json
thought: How to solve the user's problem
action: Name of the tool to be invoked
action input: Input parameters (in JSON format)
***
If no tool is needed, simply provide the answer directly.  
"""
system_message_content=system_message_template.replace("{{tool_info}}",json.dumps(TOOLS, ensure_ascii=False))
system_message={
    "role":"system",
    "content":system_message_content
}
chat_messages.append(system_message)
print(json.dumps(system_message,indent=4))
print("-"*100)
print(system_message["content"])

Notes: please replace "***" with "```" in you code.

Step 3. Function Response (Code):

import datetime
def get_current_flights_by_cities_and_date(departure_city: str, arrival_city: str, date: str):
    """
    Retrieve current flight information based on the departure city, arrival city, and search date.

    Args:
        departure_city: The city of departure, formatted as "City".
        arrival_city: The city of arrival, formatted as "City".
        date: The date of search, formatted as "yyyy-mm-dd".

    Returns:
        A dictionary of current flights with their details.
    """
    return {
        "China Eastern Airlines": {
            "airlines": "China Eastern Airlines",
            "flight_number": "MU5186",
            "departure_airport": "Pudong Airport T1",
            "arrival_airport": "Beijing Daxing Airport",
            "departure_time": "21:05",
            "arrival_time": "23:20",
            "price": "RMB 400"
        },
        "Juneyao Airlines": {
            "airlines": "Juneyao Airlines",
            "flight_number": "HO1257",
            "departure_airport": "Pudong Airport T2",
            "arrival_airport": "Beijing Daxing Airport",
            "departure_time": "21:55",
            "arrival_time": "00:30 (next day)",
            "price": "RMB 400"
        },
        "Xiamen Airlines": {
            "airlines": "Xiamen Airlines",
            "flight_number": "MF8556",
            "departure_airport": "Pudong Airport T2",
            "arrival_airport": "Beijing Daxing Airport",
            "departure_time": "20:45",
            "arrival_time": "22:55",
            "price": "RMB 490"
        },
        "China Southern Airlines": {
            "airlines": "China Southern Airlines",
            "flight_number": "CZ4451",
            "departure_airport": "Pudong Airport T2",
            "arrival_airport": "Beijing Daxing Airport",
            "departure_time": "20:45",
            "arrival_time": "22:55",
            "price": "RMB 490"
        }
    }

def book_a_flight(airlines: str, flight_number: str, departure_airport: str, arrival_airport: str, departure_time: str, arrival_time: str, price: str):
    """Book a flight by the departure city, the arrival city, and the search date.

    Args:
        airlines: the airlines
        flight_number: The flight_number
        departure_airport: The departure airport
        arrival_airport: The arrival airport
        departure_time: The departure time, in the format "yyyy-mm-dd HH:mm:ss"
        arrival_time: The arrival time, in the format "yyyy-mm-dd HH:mm:ss"
        price: The price

    Returns:
        booking status,
        booking information
    """
    return {
        "status":"success",
        "information": {
            "airlines": airlines,
            "flight_number":flight_number,
            "departure_airport":departure_airport,
            "arrival_airport":arrival_airport,
            "departure_time":departure_time,
            "arrival_time":arrival_time,
            "price": price
        }
    }

def get_current_time():
    return datetime.datetime.now()

---

Step 4. User Query (Code):

query="Please help me book a flight from Beijing to Shanghai after 2 PM."
current_time = get_current_time()
query+=" The current time: "+str(current_time)
user_info={
    "role":"user",
    "content":query
}
chat_messages.append(user_info)
model_response=model_generate(chat_messages)
assistant_conv={
    "role":"assistant",
    "content":model_response
}
print(json.dumps(assistant_conv,indent=4))
print("-"*100)
print(assistant_conv["content"])
chat_messages.append(assistant_conv)

---

Step 5. Function Calling (Code):

def decode_response(model_response):
    try:
        action = model_response.split("action: ")[1].split("\naction input: ")[0]
        action_input_str = model_response.split("\naction input: ")[1].replace("\n```","")
        action_input_json =json.loads(action_input_str)
        return True,{
            "action": action,
            "action_input": action_input_json
        }
    except:
        return False,{"error": "cannot decode the model response."}

def get_observation(status, decoded_function_call_info):
    observation_conv={
        "role":"observation",
        "content": ""
    }
    if status:
        try:
            arg_str=""
            for index,arg_key in enumerate(decoded_function_call_info["action_input"]):
                arg_value=decoded_function_call_info["action_input"][arg_key]
                if type(arg_value)==str:
                    arg_str+=arg_key+"='"+arg_value+"'"
                    if index < len(decoded_function_call_info["action_input"])-1:
                        arg_str+=", "
                else:
                    observation_conv["content"]="cannot decode the action input."
            function_call_python_str=decoded_function_call_info["action"]+"("+arg_str+")"
            function_call_response=eval(function_call_python_str)
            observation_conv["content"]=json.dumps(function_call_response, ensure_ascii=False)
        except:
            observation_conv["content"]="cannot decode the action input."
    else:
        observation_conv["content"]=decoded_function_call_info["error"]
    return observation_conv

status,decoded_function_call_info=decode_response(model_response)
observation_conv=get_observation(status,decoded_function_call_info)
print(json.dumps(observation_conv,indent=4))
print("-"*100)
print(observation_conv["content"])
chat_messages.append(observation_conv)

---

Step 6. Model Summary (Code):

model_response=model_generate(chat_messages)
assistant_conv={
    "role":"assistant",
    "content":model_response
}
print(json.dumps(assistant_conv,indent=4))
print("-"*100)
print(assistant_conv["content"])
chat_messages.append(assistant_conv)

---

Step 7. Rhetorical question and answer (Code):

query="Please help me book a ticket for China Eastern Airlines flight MU5186"
user_info={
    "role":"user",
    "content":query
}
chat_messages.append(user_info)
model_response=model_generate(chat_messages)
assistant_conv={
    "role":"assistant",
    "content":model_response
}
print(json.dumps(assistant_conv,indent=4))
print("-"*100)
print(assistant_conv["content"])
chat_messages.append(assistant_conv)

---

Step 8. Function Calling (Code):

status,decoded_function_call_info=decode_response(model_response)
observation_conv=get_observation(status,decoded_function_call_info)
print(json.dumps(observation_conv,indent=4))
print("-"*100)
print(observation_conv["content"])
chat_messages.append(observation_conv)

---

Step 9. Model Summary (Code):

model_response=model_generate(chat_messages)
assistant_conv={
    "role":"assistant",
    "content":model_response
}
print(json.dumps(assistant_conv,indent=4))
print("-"*100)
print(assistant_conv["content"])
chat_messages.append(assistant_conv)
print("-"*100)
print(chat_messages)