Go并发读写文件、分片写、分片下载文件的实现示例

目录

• 简单读取
• 读取&分片写
• 读取文件流+分片写-1
• 读取文件流+分片写-2
• 读取文件流+并发分片写
• 更好用的Copy方法
• http并发、分片下载

简单读取

func ReadFile(filePath string) (chunks []byte, err error) {
	f, err := os.Open(filePath)
	if err != nil {
		return
	}
    
    defer f.Close()
	reader := bufio.NewReader(f)

	for {
		dataByte := make([]byte, 5*1024)
		var n int
		n, err = reader.Read(dataByte)
		if err != nil || 0 == n {
			break
		}

		chunks = append(chunks, dataByte[:n]...)
		fmt.Printf("file: %s, len(chunks):%v", filePath, len(chunks))
	}

	iseoF := strings.Compare(err.Error(), "EOF")
	if isEOF == 0 {
		err = nil
		fmt.Printf("read %s success: \n, len=%v", filePath, len(chunks))
		return
	}

	fmt.Printf("readFile over")
	return
}

可以看到如文件较大，chunks会变得很大，此法只适用特定条件下的一般做法。

读取&分片写

读取文件流+分片写-1

var bufLen = 2 * 1024 * 1024

func DownLoadFileShardByFilePath1(writerFilePath string, body io.Reader) (err error) {

	f, err := os.OpenFile(writerFilePath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, os.ModeAppend)
	defer f.Close()
	if err != nil {
		fmt.Println("open err:" + err.Error())
		return
	}

	writer := bufio.NewWriter(f)
	bs := make([]byte, bufLen)
	for {
		var read int
		read, err = body.Read(bs)
		if err != nil || 0 == read {
			break
		}

		_, err = writer.Write(bs[:read])
		if err != nil {
			fmt.Println("write err:" + err.Error())
			break
		}
	}

	if err == io.EOF {
		err = nil
	}

	if err != nil {
		return
	}

	if err = writer.Flush(); err != nil {
		fmt.Println("writer flush err: ", err.Error())
		return
	}

	fmt.Printf("downLoad over")
	return
}

读取文件流+分片写-2

var bufLen = 2 * 1024 * 1024

func DownLoadFileShard(writerFilePath string, body io.Reader) (err error) {

	f, err := os.OpenFile(writerFilePath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, os.ModeAppend)
	if err != nil {
		fmt.Println("open err:" + err.Error())
		return
	}

    defer f.Close()
	bs := make([]byte, bufLen)
	writer := bufio.NewWriter(f)
	for {
		var read int
		switch read, err = body.Read(bs[:]); true {
		case read < 0:
			fmt.Println("read err: ", err.Error())
			return
		case read == 0, err == io.EOF:
			fmt.Printf("downLoad over")
			return writer.Flush()
		case read > 0:
			_, err = writer.Write(bs[:read])
			if err != nil {
				fmt.Println("write err:" + err.Error())
				return
			}
		}
	}

	return
}

读取文件流+并发分片写

type FileShard struct {
	Data []byte
	Err  error
	Code int // 0-正常  -1=失败
}

var bufLen = 2 * 1024 * 1024

func DownLoadFileShardCon(writerFilePath string, body io.Reader) (err error) {

	writerFile, err := os.OpenFile(writerFilePath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, os.ModeAppend)
	if err != nil {
		fmt.Println("open err:" + err.Error())
		return
	}

    defer writerFile.Close()
	ch, complete := make(chan *FileShard), make(chan struct{})
	go func() {
		writer := bufio.NewWriter(writerFile)

	youKnow:
		for {
			select {
			case data := <-ch:
				if data == nil {
					err = writer.Flush()
					break youKnow
				}

				if data.Code != 0 {
					err = data.Err
					break youKnow
				}

				if _, err = writer.Write(data.Data); err != nil {
					fmt.Println("write err:", err.Error())
				}
			}
		}

		close(complete)
	}()

	go func() {
		bs := make([]byte, bufLen)
		for {
			switch read, readErr := body.Read(bs[:]); true {
			case read < 0:
				ch <- &FileShard{Code: -1, Err: readErr}
				close(ch)
				return
			case read == 0, err == io.EOF:
				close(ch)
				return
			case read > 0:
				ch <- &FileShard{Data: bs[:read], Code: 0}
			}
		}

	}()

	select {
	case <-complete:
		break
	}

	fmt.Printf("downLoad over")
	return
}

并发思路有很多种，看你代码怎么写哦，条条大路通罗马！

更好用的Copy方法

要提醒的是，还有一个很不错的方法在io包里，就是io.Copy()，可防止大文件处理时的内存溢出，也可以替换上述主要流程，比如：

func IOCopyExample(writerFilePath string, body io.Reader) (err error) {
	f, err := os.OpenFile(writerFilePath, os.O_CREATE|os.O_TRUNC, 0666)
	if err != nil {
		return
	}
	
	defer f.Close()
	writer := bufio.NewWriter(f)
	_, err = io.Copy(writer, body)
	_ = writer.Flush()
	return
}

http并发、分片下载

基于http的Range来完成：

func DownloadFileRangeandroid(url, writeFile string) error {
	f, err := os.OpenFile(writeFile , os.O_CREATE|os.O_TRUNC, 0666)
	if err != nil {
		return err
	}

	defer f.Close()

	resp, err := http.Head(url)
	if err != nil {
		return err
	}

	size, err := strconv.Atoi(resp.Header.Get("Content-Length"))
	if err != nil {
		return err
	}

	con := getSize(size)  // getSize函数用来计算每次的并发数，可按自己方式自行指定
	var startandroid, end int64
	for i := 0; i < con; i++ {
		
		start = int64(i) * int64(size/con)
		end = sphptart + int64(size/con) - 1

		go f编程客栈unc(n int, offset, end int64) {
			req := &http.Request{}
			req, err = http.NewRequest(http.MethodGet, url, nil)
			req.Header.Set("Range", fmt.Sprintf("bytes=%v-%v", offset, end))

			client := &http.Client{}
			resp, err = client.Do(req)
			if err != nil {
				return
			}

			defer resp.Body.Close()

			f.Seek(offset, 0)
			_, err = io.Copy(f, resp.Body)
			if err != nil {
				// log
			}

		}(i, start, end)
	}

	return nil
}

到此这篇关于Go并发读写文件、分片写、分片下载文件的实现示例的文章就介绍到这了,更多相关Go并发读写、分片写、分python片下载内容请搜索编程客栈(www.devze.com)以前的文章或继续浏览下面的相关文章希望大家以后多多支持编程客栈(www.devze.com)！