倾心分享，如有帮助，欢迎留下足迹~
目录
简单读取
func ReadFile(filePath string) (chunks []byte, err error) {
f, err := os.Open(filePath)
if err != nil {
return
}
defer f.Close()
reader := bufio.NewReader(f)
for {
dataByte := make([]byte, 5*1024)
var n int
n, err = reader.Read(dataByte)
if err != nil || 0 == n {
break
}
chunks = append(chunks, dataByte[:n]...)
fmt.Printf("file: %s, len(chunks):%v", filePath, len(chunks))
}
isEOF := strings.Compare(err.Error(), "EOF")
if isEOF == 0 {
err = nil
fmt.Printf("read %s success: \n, len=%v", filePath, len(chunks))
return
}
fmt.Printf("readFile over")
return
}可以看到如文件较大，chunks会变得很大，此法只适用特定条件下的一般做法。
读取&分片写
读取文件流+分片写-1
var bufLen = 2 * 1024 * 1024
func DownLoadFileShardByFilePath1(writerFilePath string, body io.Reader) (err error) {
f, err := os.OpenFile(writerFilePath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, os.ModeAppend)
defer f.Close()
if err != nil {
fmt.Println("open err:" + err.Error())
return
}
writer := bufio.NewWriter(f)
bs := make([]byte, bufLen)
for {
var read int
read, err = body.Read(bs)
if err != nil || 0 == read {
break
}
_, err = writer.Write(bs[:read])
if err != nil {
fmt.Println("write err:" + err.Error())
break
}
}
if err == io.EOF {
err = nil
}
if err != nil {
return
}
if err = writer.Flush(); err != nil {
fmt.Println("writer flush err: ", err.Error())
return
}
fmt.Printf("downLoad over")
return
}读取文件流+分片写-2
var bufLen = 2 * 1024 * 1024
func DownLoadFileShard(writerFilePath string, body io.Reader) (err error) {
f, err := os.OpenFile(writerFilePath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, os.ModeAppend)
if err != nil {
fmt.Println("open err:" + err.Error())
return
}
defer f.Close()
bs := make([]byte, bufLen)
writer := bufio.NewWriter(f)
for {
var read int
switch read, err = body.Read(bs[:]); true {
case read < 0:
fmt.Println("read err: ", err.Error())
return
case read == 0, err == io.EOF:
fmt.Printf("downLoad over")
return writer.Flush()
case read > 0:
_, err = writer.Write(bs[:read])
if err != nil {
fmt.Println("write err:" + err.Error())
return
}
}
}
return
}读取文件流+并发分片写
type FileShard struct {
Data []byte
Err error
Code int // 0-正常 -1=失败
}
var bufLen = 2 * 1024 * 1024
func DownLoadFileShardCon(writerFilePath string, body io.Reader) (err error) {
writerFile, err := os.OpenFile(writerFilePath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, os.ModeAppend)
if err != nil {
fmt.Println("open err:" + err.Error())
return
}
defer writerFile.Close()
ch, complete := make(chan *FileShard), make(chan struct{})
go func() {
writer := bufio.NewWriter(writerFile)
youKnow:
for {
select {
case data := <-ch:
if data == nil {
err = writer.Flush()
break youKnow
}
if data.Code != 0 {
err = data.Err
break youKnow
}
if _, err = writer.Write(data.Data); err != nil {
fmt.Println("write err:", err.Error())
}
}
}
close(complete)
}()
go func() {
bs := make([]byte, bufLen)
for {
switch read, readErr := body.Read(bs[:]); true {
case read < 0:
ch <- &FileShard{Code: -1, Err: readErr}
close(ch)
return
case read == 0, err == io.EOF:
close(ch)
return
case read > 0:
ch <- &FileShard{Data: bs[:read], Code: 0}
}
}
}()
select {
case <-complete:
break
}
fmt.Printf("downLoad over")
return
}并发思路有很多种，看你代码怎么写哦，条条大路通罗马！
更好用的Copy方法
要提醒的是，还有一个很不错的方法在io包里，就是io.Copy()，可防止大文件处理时的内存溢出，也可以替换上述主要流程，比如：
func IOCopyExample(writerFilePath string, body io.Reader) (err error) {
f, err := os.OpenFile(writerFilePath, os.O_CREATE|os.O_TRUNC, 0666)
if err != nil {
return
}
defer f.Close()
writer := bufio.NewWriter(f)
_, err = io.Copy(writer, body)
_ = writer.Flush()
return
}http并发、分片下载
基于http的Range来完成：
func DownloadFileRange(url, writeFile string) error {
f, err := os.OpenFile(writeFile , os.O_CREATE|os.O_TRUNC, 0666)
if err != nil {
return err
}
defer f.Close()
resp, err := http.Head(url)
if err != nil {
return err
}
size, err := strconv.Atoi(resp.Header.Get("Content-Length"))
if err != nil {
return err
}
con := getSize(size) // getSize函数用来计算每次的并发数，可按自己方式自行指定
var start, end int64
for i := 0; i < con; i++ {
start = int64(i) * int64(size/con)
end = start + int64(size/con) - 1
go func(n int, offset, end int64) {
req := &http.Request{}
req, err = http.NewRequest(http.MethodGet, url, nil)
req.Header.Set("Range", fmt.Sprintf("bytes=%v-%v", offset, end))
client := &http.Client{}
resp, err = client.Do(req)
if err != nil {
return
}
defer resp.Body.Close()
f.Seek(offset, 0)
_, err = io.Copy(f, resp.Body)
if err != nil {
// log
}
}(i, start, end)
}
return nil
}
