Home AMX User Forum AMX General Discussion

REMOVE_STRING Best practice

Help.

I guess I still don't have the best understanding of how to effectively parse a buffer without destroying it. I thought the buffer remained the same as you pass through each condition but that doesn't seem the case here. The first variable works, the rest does not. Any advice would be great. Thanks.

string:
{"current_time": 1426595891, "items": [{ "bus": "bus1","stop": "lobby7","modifier": "","time": 1426595881.08877}, { "bus": "bus1","stop": "lobby1","modifier": "","time": 1426595981.6541}, { "bus": "bus1","stop": "ss2mp","modifier": "","time": 1426596043.26716}, { "bus": "bus1","stop": "sl2mp","modifier": "","time": 1426596105.19317}, { "bus": "bus1","stop": "mp1","modifier": "","time": 1426596203.61697}, { "bus": "bus1","stop": "ss2mp5","modifier": "","time": 1426596255.84008}, { "bus": "bus1","stop": "mp5","modifier": "","time": 1426596283.88472}, { "bus": "bus1","stop": "mp4","modifier": "","time": 1426596365.64478}, { "bus": "bus1","stop": "mp6","modifier": "","time": 1426596457.76979}, { "bus": "bus1","stop": "sl2l7","modifier": "","time": 1426596534.33809}, { "bus": "bus5","stop": "lobby1south","modifier": "","time": 1426596043.43202}, { "bus": "bus5","stop": "b26","modifier": "","time": 1426596114.69907}, { "bus": "bus5","stop": "ss2b23","modifier": "","time": 1426596138.91473}, { "bus": "bus5","stop": "b23","modifier": "","time": 1426596239.65891}, { "bus": "bus5","stop": "ss2b200","modifier": "","time": 1426596318.22645}, { "bus": "bus5","stop": "sl2b200","modifier": "","time": 1426596400.11884}, { "bus": "bus5","stop": "b200","modifier": "","time": 1426595912.1329}, { "bus": "bus5","stop": "ss2l1","modifier": "","time": 1426595974.02958}, { "bus": "bus2","stop": "mp6","modifier": "","time": 1426594740}, { "bus": "bus2","stop": "lobby7","modifier": "","time": 1426594920}, { "bus": "bus2","stop": "lobby1","modifier": "","time": 1426595040}, { "bus": "bus2","stop": "b200","modifier": "","time": 1426595160}], "buses": {"bus1": {"lat": 39.164114,"lon": -76.896467,"loopTime": 754.3380870042, "stop": "lobby7"}, "bus5": {"lat": 39.159872,"lon": -76.898317,"loopTime": 548.118844121}, "bus2": {"lat": 39.170054,"lon": -76.89581,"loopTime": 0}, "bus4": {"lat": 39.167839,"lon": -76.898051,"loopTime": 0}}}

parse function:
DEFINE_FUNCTION fnPARSE_BUS_BUFFER()
{
    LOCAL_VAR CHAR FULL[50]
    LOCAL_VAR CHAR EXPRESS[50]
    LOCAL_VAR CHAR SOUTH[50]
    LOCAL_VAR INTEGER B1LOBBY7
    LOCAL_VAR INTEGER B2LOBBY7
    LOCAL_VAR INTEGER B1MP6
    LOCAL_VAR INTEGER B2MP6
    LOCAL_VAR INTEGER MP1
    LOCAL_VAR INTEGER MP4
    LOCAL_VAR INTEGER B1LOBBY1
    LOCAL_VAR INTEGER B2LOBBY1
    LOCAL_VAR INTEGER BL200

    LOCAL_VAR CHAR cCURRENT_TIME[20]

    WHILE(FIND_STRING(cBUS_BUFFER, "'{"current_time": '", 1))
    {
	cCURRENT_TIME = REMOVE_STRING(cBUS_BUFFER, '{"current_time": ', 1)
	cCURRENT_TIME = LEFT_STRING(cBUS_BUFFER, 10)
		
		IF(FIND_STRING(cBUS_BUFFER, "'{ "bus": "bus1","stop": "lobby7","modifier": "","time": '", 1)) //Bus 1, Lobby 7
		{
			FULL = REMOVE_STRING(cBUS_BUFFER, "'{ "bus": "bus1","stop": "lobby7","modifier": "","time": '", 1)
			FULL = LEFT_STRING(cBUS_BUFFER, 16)
			B1LOBBY7 = fnCONVERT_BUS_TIME(FULL, cCURRENT_TIME)
    
			IF(B1LOBBY7 > 0)
			{
			    uBUSES[1].FULL_ETA = "ITOA(B1LOBBY7), ' Minutes'"
			}
			ELSE IF(B1LOBBY7 == 0)
			{
			    uBUSES[1].FULL_ETA = "'BOARDING'"
			}
		}
		ELSE IF(FIND_STRING(cBUS_BUFFER, "'{ "bus": "bus1","stop": "lobby1","modifier": "","time": '", 1))//Bus 1, Lobby 1
		{
			FULL = REMOVE_STRING(cBUS_BUFFER, "'{ "bus": "bus1","stop": "lobby1","modifier": "","time": '", 1)
			FULL = LEFT_STRING(cBUS_BUFFER, 16)
			B1LOBBY1 = fnCONVERT_BUS_TIME(FULL, cCURRENT_TIME)
    
			IF(B1LOBBY1 > 0)
			{
			    uBUSES[2].FULL_ETA = "ITOA(B1LOBBY1), ' Minutes'"
			}
			ELSE IF(B1LOBBY1 == 0)
			{
			    uBUSES[2].FULL_ETA = "'BOARDING'"
			}
		}
		ELSE IF(FIND_STRING(cBUS_BUFFER, "'{ "bus": "bus1","stop": "mp1","modifier": "","time": '", 1)) //Bus 1, MP1
		{
			FULL = REMOVE_STRING(cBUS_BUFFER, "'{ "bus": "bus1","stop": "mp1","modifier": "","time": '", 1)
			FULL = LEFT_STRING(cBUS_BUFFER, 16)
			MP1 = fnCONVERT_BUS_TIME(FULL, cCURRENT_TIME)
    
			IF(MP1 > 0)
			{
			    uBUSES[3].FULL_ETA = "ITOA(MP1), ' Minutes'"
			}
			ELSE IF(MP1 == 0)
			{
			    uBUSES[3].FULL_ETA = "'BOARDING'"
			}
		}
    
		ELSE IF(FIND_STRING(cBUS_BUFFER, "'{ "bus": "bus1","stop": "mp4","modifier": "","time": '", 1)) //Bus 1, MP4
		{
			FULL = REMOVE_STRING(cBUS_BUFFER, "'{ "bus": "bus1","stop": "mp4","modifier": "","time": '", 1)
			FULL = LEFT_STRING(cBUS_BUFFER, 16)
			MP4 = fnCONVERT_BUS_TIME(FULL, cCURRENT_TIME)
    
			IF(MP4 > 0)
			{
			    uBUSES[4].FULL_ETA = "ITOA(MP4), ' Minutes'"
			}
			ELSE IF(MP4 == 0)
			{
			    uBUSES[4].FULL_ETA = "'BOARDING'"
			}
		}
		ELSE IF(FIND_STRING(cBUS_BUFFER, "'{ "bus": "bus1","stop": "mp6","modifier": "","time": '", 1)) //Bus 1, MP6
		{
			FULL = REMOVE_STRING(cBUS_BUFFER, "'{ "bus": "bus1","stop": "mp6","modifier": "","time": '", 1)
			FULL = LEFT_STRING(cBUS_BUFFER, 16)
			B1MP6 = fnCONVERT_BUS_TIME(FULL, cCURRENT_TIME)
    
			IF(B1MP6 > 0)
			{
			    uBUSES[5].FULL_ETA = "ITOA(B1MP6), ' Minutes'"
			}
			ELSE IF(B1MP6 == 0)
			{
			    uBUSES[5].FULL_ETA = "'BOARDING'"
			}
		}
		ELSE IF(FIND_STRING(cBUS_BUFFER, "'{ "bus": "bus2","stop": "lobby7","modifier": "","time": '", 1)) //Bus 2, Lobby 7
		{
			EXPRESS = REMOVE_STRING(cBUS_BUFFER, "'{ "bus": "bus2","stop": "lobby7","modifier": "","time": '", 1)
			EXPRESS = LEFT_STRING(cBUS_BUFFER, 16)
			B2LOBBY7 = fnCONVERT_BUS_TIME(EXPRESS, cCURRENT_TIME)
    
			IF(B2LOBBY7 > 0)
			{
			    uBUSES[1].EXPRESS_ETA = "ITOA(B2LOBBY7), ' Minutes'"
			}
			ELSE IF(B2LOBBY7 == 0)
			{
			    uBUSES[1].EXPRESS_ETA = "'BOARDING'"
			}
		}
		ELSE IF(FIND_STRING(cBUS_BUFFER, "'{ "bus": "bus2","stop": "lobby1","modifier": "","time": '", 1)) //Bus 2, Lobby 1
		{
			EXPRESS = REMOVE_STRING(cBUS_BUFFER, "'{ "bus": "bus2","stop": "lobby1","modifier": "","time": '", 1)
			EXPRESS = LEFT_STRING(cBUS_BUFFER, 16)
			B2LOBBY1 = fnCONVERT_BUS_TIME(EXPRESS, cCURRENT_TIME)
    
			IF(B2LOBBY1 > 0)
			{
			    uBUSES[2].EXPRESS_ETA = "ITOA(B2LOBBY1), ' Minutes'"
			}
			ELSE IF(B2LOBBY1 == 0)
			{
			    uBUSES[2].EXPRESS_ETA = "'BOARDING'"
			}
		}
		ELSE IF(FIND_STRING(cBUS_BUFFER, "'{ "bus": "bus2","stop": "b200","modifier": "","time": '", 1)) //Bus 2, Building 200
		{
			EXPRESS = REMOVE_STRING(cBUS_BUFFER, "'{ "bus": "bus2","stop": "b200","modifier": "","time": '", 1)
			EXPRESS = LEFT_STRING(cBUS_BUFFER, 16)
			BL200 = fnCONVERT_BUS_TIME(EXPRESS, cCURRENT_TIME)
    
			IF(BL200 > 0)
			{
			    uBUSES[6].EXPRESS_ETA = "ITOA(BL200), ' Minutes'"
			}
			ELSE IF(BL200 == 0)
			{
			    uBUSES[6].EXPRESS_ETA = "'BOARDING'"
			}
		}
		ELSE IF(FIND_STRING(cBUS_BUFFER, "'{ "bus": "bus2","stop": "mp6","modifier": "","time": '", 1)) //Bus 2, MP6
		{
			EXPRESS = REMOVE_STRING(cBUS_BUFFER, "'{ "bus": "bus2","stop": "mp6","modifier": "","time": '", 1)
			EXPRESS = LEFT_STRING(cBUS_BUFFER, 16)
			B2MP6 = fnCONVERT_BUS_TIME(EXPRESS, cCURRENT_TIME)
    
			IF(B2MP6 > 0)
			{
			    uBUSES[5].EXPRESS_ETA = "ITOA(B2MP6), ' Minutes'"
			}
			ELSE IF(B2MP6 == 0)
			{
			    uBUSES[5].EXPRESS_ETA = "'BOARDING'"
			}
		}
	    }
    }

Comments

  • JasonSJasonS Posts: 229
    Where is the code that calls the parse function? What might be happening is that a string that long is received across multiple data_events. If the parse function is called on the first data_event but the rest of the string is not received yet, the "header" is removed from the buffer, but none of your if find_string cases evaluate because that part of the string has not been received yet. You have already removed the "header" from the string so the parse function doesn't do anything on successive calls because it does not find the "header" until a whole new string comes in.

    if you want to parse the whole string in a loop I would suggest removing from the buffer based on the end of the string, possibly "}}}". That way you know you have the whole string before you start parsing.
  • JasonS wrote: »
    Where is the code that calls the parse function? What might be happening is that a string that long is received across multiple data_events. If the parse function is called on the first data_event but the rest of the string is not received yet, the "header" is removed from the buffer, but none of your if find_string cases evaluate because that part of the string has not been received yet. You have already removed the "header" from the string so the parse function doesn't do anything on successive calls because it does not find the "header" until a whole new string comes in.

    if you want to parse the whole string in a loop I would suggest removing from the buffer based on the end of the string, possibly "}}}". That way you know you have the whole string before you start parsing.

    Hello Jason,

    I am calling the function in the Data_Event for the bus port, under string... is this not the best application? I got it to work now, but it took a few minutes after coming online to start parsing correctly.
  • ericmedleyericmedley Posts: 4,177
    This is probably a good thing to do in almost all cases. I think we tend to believe and thus rely on the idea that data.text has some kind of magic mojo that somehow knows when an incoming string is complete. This is not really the case. The fact of the matter is that form "most" things it works pretty well. But, there are times when it all falls apart; those being abnormally large strings, strings with seemingly multiple terminators, sputtery or overly chatty devices.

    I've treid to get into the habit of going ahead and building my own string buffers and not relying upon data.text and/or create_buffer to tell me when a string is done. It's really pretty simple to do, it doesn't add that many lines of code and is waaaaay more reliable.

    A simple way to do this is to just tag any incmoing data.text stuff on the end of a buffer and quickly search for the end delimiter. Here's a simple example:
    data_event[MySillyString_Device]]
    	string:{
    		MySillyString_Buffer="MySillyString_Buffer,data.text"
    		while(find_string(MySillyString_Buffer,13,1))){
    			fn_Parse_My_String(remove_string(MySillyString_Buffer,13,1));
    			}
    		}
    	}// d_e
    

    Here again, not exactly how I'd do it but the concept is simple.

    keep adding any strings coming onto the end of the buffer.
    Then sit and spin on the while loop, peeling off messages that end in the delimiter. (In this example, the end of message is a LineFeed (13)). So, just peel off a message and process.

    If your sputtery device sends something like:

    " Message One<LF> Message Two <LF> Mes"

    and then sputters but finishes with...
    "sage Three<LF>"

    This example might generate two data events.

    If you just relaied upon data.text it could get goofed up. Depending upon how you code you might even ignore the second message and the third message will not parse correctly. If seen this a lot in code I get.

    So, I'd suggest bulding your own routine to put the buffer together that doesn't rely on the timing of either the device or the AMX master's idea of what a message should look like.
  • JasonSJasonS Posts: 229
    mjones2620 wrote: »
    Hello Jason,

    I am calling the function in the Data_Event for the bus port, under string... is this not the best application? I got it to work now, but it took a few minutes after coming online to start parsing correctly.

    Yes that is where I would call the function. I was just verifying that is where it was being called for my own thought process.

    If you do use a while loop to process your buffer (I generally do) make sure there is a way to exit that loop if you don't get the results you expect or the string is not long enough (usually this is more of a concern with a protocol which uses a length byte and a checksum as opposed to a clear delimiter). An endless while loop will severely affect processor performance!
  • ericmedleyericmedley Posts: 4,177
    JasonS wrote: »
    Yes that is where I would call the function. I was just verifying that is where it was being called for my own thought process.

    If you do use a while loop to process your buffer (I generally do) make sure there is a way to exit that loop if you don't get the results you expect or the string is not long enough (usually this is more of a concern with a protocol which uses a length byte and a checksum as opposed to a clear delimiter). An endless while loop will severely affect processor performance!

    Back in those halcyon days when dinosaurs roamed the Earth (When I took Prog II class) we were told them not to use the While loop for the very reason that it was possible to lock yourself up. I do use it as well; gamling on it not locking me up. Iv'e yet to be bitten by it. But I do need to make good n sure I'll never get bit by it.

    In casese where I'm not so sure I go ahead and create a really fast timeline that can run endlessly if need be but won't lock me up. I can then watch the thing and if it's running on and on, I can just kill it and clear the messy buffer.
  • TonyAngeloTonyAngelo Posts: 315
    Since you're parsing JSON, here is a function I use for getting values for JSON keys, it won't do the parsing by itself, but it may help you down the correct path.
    DEFINE_FUNCTION CHAR[500] fnJSONGetKey(CHAR sArgJSON[], CHAR sArgKEY[])
    {(* ////////////////////////////////////////////////////////////////////////////////////////
        This function returns the JSON Value associated with the key sArgKEY from the JSON data sArgJSON
        
        This function is non-destructive to the data inputs
        ///////////////////////////////////////////////////////////////////////////////////////
      *)
        STACK_VAR INTEGER nMyBegin
        STACK_VAR INTEGER nMyValueLength
        STACK_VAR INTEGER nMyKeyLength
        STACK_VAR CHAR sMyValue[500]
        
        // get the length of the key
        nMyKeyLength=LENGTH_STRING(sArgKEY)+3
        // get the begning of the value
        nMyBegin=FIND_STRING(sArgJSON,"'"',sArgKEY,'":'",1)
        // if the key exists
        IF(nMyBegin)
        {
    	// get the length of the value
    	nMyValueLength=FIND_STRING(sArgJSON,',',nMyBegin+nMyKeyLength)
    	// get the value
    	sMyValue = MID_STRING(sArgJSON,nMyBegin+nMyKeyLength,nMyValueLength-nMyBegin-nMyKeyLength)
    	// if the value is not a key/value pair itself
    	IF(!FIND_STRING(sMyValue,'{',1))
    	{
    	    // if it was the last value of an array, it will have a closing bracket
    	    IF(FIND_STRING(sMyValue,']',1))
    	    {
    		// get rid of the closing bracket
    		SET_LENGTH_STRING(sMyValue,LENGTH_STRING(sMyValue)-1)
    	    }
    	    // if it was the last value it will have a closing bracket
    	    IF(FIND_STRING(sMyValue,'}',1))
    	    {
    		// get rid of the closing bracket
    		SET_LENGTH_STRING(sMyValue,LENGTH_STRING(sMyValue)-1)
    	    }
    	    // text string with quotes around it
    	    IF(FIND_STRING(sMyValue,'"',1))
    	    {
    		STACK_VAR INTEGER nMyLength
    		// get the length of the value
    		nMyLength=LENGTH_STRING(sMyValue)
    		// get the 2nd through 2nd to last characters
    		sMyValue=MID_STRING(sMyValue,2,nMyLength-2)
    	    }
    	}
        }
        // return the value
        RETURN sMyValue;
    }
    
Sign In or Register to comment.